From a30b731e4f6df2f4985d2554778ae0c8603ba896 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Fri, 30 Oct 2026 07:22:28 +0100 Subject: [PATCH 01/82] Update llama.cpp to v0.8.0, decode through its ext batch API, and refuse pooled logits, invalid sequences, and oversized DRY breakers --- .../src/bindgen_config.rs | 6 + llama-cpp-bindings-build/src/cmake_config.rs | 2 +- llama-cpp-bindings-sys/Cargo.toml | 3 +- llama-cpp-bindings-sys/llama.cpp | 2 +- llama-cpp-bindings-sys/wrapper.h | 1 + llama-cpp-bindings-sys/wrapper_batch.cpp | 111 +++ llama-cpp-bindings-sys/wrapper_batch.h | 58 ++ llama-cpp-bindings-sys/wrapper_chat_apply.cpp | 2 +- llama-cpp-bindings-sys/wrapper_chat_parse.cpp | 2 +- llama-cpp-bindings-sys/wrapper_common.cpp | 23 +- llama-cpp-bindings-sys/wrapper_common.h | 8 +- llama-cpp-bindings-sys/wrapper_context.cpp | 4 - llama-cpp-bindings-sys/wrapper_context.h | 2 - llama-cpp-bindings-sys/wrapper_gbnf.cpp | 2 +- llama-cpp-bindings-sys/wrapper_gbnf.h | 2 +- llama-cpp-bindings-sys/wrapper_reasoning.cpp | 6 +- .../src/classify_sample_loop.rs | 6 +- .../src/prime_kv_cache.rs | 10 +- .../src/prime_kv_cache_with.rs | 7 +- .../tests/batch_context_limits.rs | 42 ++ .../tests/context_after_decoding_text.rs | 6 +- .../tests/context_state.rs | 162 ++--- .../tests/embedding_models.rs | 45 +- .../tests/generation_control.rs | 101 ++- .../tests/hidden_states.rs | 40 +- .../tests/logits_availability.rs | 80 +++ llama-cpp-bindings-tests/tests/main.rs | 2 + .../tests/model_introspection.rs | 10 +- .../tests/multimodal_audio.rs | 13 +- .../tests/multimodal_fusion.rs | 20 +- .../tests/multimodal_vision.rs | 192 ++++-- .../tests/structured_chat_output.rs | 106 +-- .../src/batch_capacity.rs | 109 --- .../src/batch_capacity_error.rs | 7 - llama-cpp-bindings-types/src/lib.rs | 4 - llama-cpp-bindings/src/batch_add_error.rs | 28 +- llama-cpp-bindings/src/context.rs | 392 ++++++++--- .../src/context/context_limits.rs | 152 ++++ .../src/context/decoded_output_rows.rs | 39 ++ .../src/context/decoded_outputs.rs | 135 ++++ llama-cpp-bindings/src/context/params.rs | 20 + .../src/dry_sequence_breakers.rs | 139 ++++ llama-cpp-bindings/src/error.rs | 2 + .../src/error/batch_transfer_error.rs | 20 + llama-cpp-bindings/src/error/decode_error.rs | 4 + llama-cpp-bindings/src/error/encode_error.rs | 4 + llama-cpp-bindings/src/error/fit_error.rs | 2 + llama-cpp-bindings/src/error/grammar_error.rs | 2 - .../src/error/json_schema_to_grammar_error.rs | 2 + llama-cpp-bindings/src/error/logits_error.rs | 12 +- llama-cpp-bindings/src/error/sample_error.rs | 3 + .../src/error/sampling_error.rs | 22 + .../src/error/sequence_id_out_of_range.rs | 6 + .../src/error/token_outside_vocabulary.rs | 14 + .../src/json_schema_to_grammar.rs | 35 +- llama-cpp-bindings/src/lib.rs | 2 + llama-cpp-bindings/src/llama_batch.rs | 649 +++++------------- llama-cpp-bindings/src/llama_batch_entry.rs | 11 + llama-cpp-bindings/src/model.rs | 8 +- llama-cpp-bindings/src/model/params.rs | 34 + llama-cpp-bindings/src/model/vocab_type.rs | 61 +- llama-cpp-bindings/src/mtmd.rs | 7 +- .../src/mtmd/fitted_mtmd_input_chunk.rs | 21 +- .../src/mtmd/micro_batch_tokens.rs | 10 - .../src/mtmd/mtmd_capability.rs | 15 + llama-cpp-bindings/src/mtmd/mtmd_context.rs | 105 ++- .../src/mtmd/mtmd_eval_error.rs | 3 + .../src/mtmd/mtmd_eval_limits.rs | 115 ++++ .../src/mtmd/mtmd_input_chunks.rs | 30 +- .../src/mtmd/positive_batch_tokens.rs | 37 - llama-cpp-bindings/src/sampled_token.rs | 34 + .../src/sampled_token_classifier.rs | 16 +- llama-cpp-bindings/src/sampling.rs | 21 +- llama-cpp-bindings/src/timing.rs | 264 +++---- llama-cpp-gbnf/src/gbnf_validation_error.rs | 6 +- llama-cpp-gbnf/src/validate_gbnf.rs | 23 +- ...wrapper_allocated_error_message_pointer.rs | 3 +- .../src/wrapper_sources.rs | 1 + 78 files changed, 2291 insertions(+), 1414 deletions(-) create mode 100644 llama-cpp-bindings-sys/wrapper_batch.cpp create mode 100644 llama-cpp-bindings-sys/wrapper_batch.h create mode 100644 llama-cpp-bindings-tests/tests/batch_context_limits.rs create mode 100644 llama-cpp-bindings-tests/tests/logits_availability.rs delete mode 100644 llama-cpp-bindings-types/src/batch_capacity.rs delete mode 100644 llama-cpp-bindings-types/src/batch_capacity_error.rs create mode 100644 llama-cpp-bindings/src/context/context_limits.rs create mode 100644 llama-cpp-bindings/src/context/decoded_output_rows.rs create mode 100644 llama-cpp-bindings/src/context/decoded_outputs.rs create mode 100644 llama-cpp-bindings/src/dry_sequence_breakers.rs create mode 100644 llama-cpp-bindings/src/error/batch_transfer_error.rs create mode 100644 llama-cpp-bindings/src/error/sequence_id_out_of_range.rs create mode 100644 llama-cpp-bindings/src/llama_batch_entry.rs delete mode 100644 llama-cpp-bindings/src/mtmd/micro_batch_tokens.rs create mode 100644 llama-cpp-bindings/src/mtmd/mtmd_capability.rs create mode 100644 llama-cpp-bindings/src/mtmd/mtmd_eval_limits.rs delete mode 100644 llama-cpp-bindings/src/mtmd/positive_batch_tokens.rs diff --git a/llama-cpp-bindings-build/src/bindgen_config.rs b/llama-cpp-bindings-build/src/bindgen_config.rs index bb899a456..ef14c0ab2 100644 --- a/llama-cpp-bindings-build/src/bindgen_config.rs +++ b/llama-cpp-bindings-build/src/bindgen_config.rs @@ -11,6 +11,8 @@ use crate::target_os::TargetOs; const DEPRECATED_FUNCTIONS: &[&str] = &[ "ggml_add1", "ggml_add1_inplace", + "ggml_flash_attn_ext_set_prec", + "ggml_mul_mat_set_prec", "ggml_rope_custom", "ggml_rope_custom_inplace", "ggml_type_sizef", @@ -52,6 +54,8 @@ const DEPRECATED_FUNCTIONS: &[&str] = &[ "llama_token_sep", "llama_vocab_cls", "mtmd_encode", + "mtmd_helper_model_can_chat", + "mtmd_helper_support_video", "mtmd_image_tokens_get_nx", "mtmd_image_tokens_get_ny", ]; @@ -182,9 +186,11 @@ fn create_base_builder(llama_src: &Path, callbacks: BindingCallbacks) -> bindgen .allowlist_function("llama_.*") .allowlist_type("llama_.*") .allowlist_var("LLAMA_TOKEN_NULL") + .allowlist_var("LLAMA_DRY_MAX_SEQ_BREAKERS") .allowlist_function("mtmd_.*") .allowlist_type("mtmd_.*") .blocklist_function("ggml_fopen") + .blocklist_function("llama_adapter_lora_init_from_file_ptr") .blocklist_function("gguf_init_from_file_ptr") .blocklist_function("gguf_write_to_file_ptr") .blocklist_function("llama_model_load_from_file_ptr") diff --git a/llama-cpp-bindings-build/src/cmake_config.rs b/llama-cpp-bindings-build/src/cmake_config.rs index 742b05238..768d78a26 100644 --- a/llama-cpp-bindings-build/src/cmake_config.rs +++ b/llama-cpp-bindings-build/src/cmake_config.rs @@ -76,7 +76,7 @@ fn configure_base_defines(config: &mut Config) { config.define("LLAMA_BUILD_COMMON", "ON"); config.define("LLAMA_BUILD_MTMD", "ON"); config.define("MTMD_VIDEO", "OFF"); - config.define("LLAMA_CURL", "OFF"); + config.define("LLAMA_OPENSSL", "OFF"); } fn configure_cpu_features( diff --git a/llama-cpp-bindings-sys/Cargo.toml b/llama-cpp-bindings-sys/Cargo.toml index b4ce0791e..29c9cfeea 100644 --- a/llama-cpp-bindings-sys/Cargo.toml +++ b/llama-cpp-bindings-sys/Cargo.toml @@ -56,14 +56,15 @@ include = [ "/llama.cpp/vendor", "/llama.cpp/CMakeLists.txt", + "/llama.cpp/LICENSE", "/llama.cpp/common/CMakeLists.txt", + "/llama.cpp/common/parsers/sources.cmake", "/llama.cpp/ggml/CMakeLists.txt", "/llama.cpp/ggml/src/CMakeLists.txt", "/llama.cpp/src/CMakeLists.txt", "/llama.cpp/cmake", "/llama.cpp/ggml/cmake", - "/llama.cpp/common/cmake", ] [dependencies] diff --git a/llama-cpp-bindings-sys/llama.cpp b/llama-cpp-bindings-sys/llama.cpp index 5266f24da..8642541b2 160000 --- a/llama-cpp-bindings-sys/llama.cpp +++ b/llama-cpp-bindings-sys/llama.cpp @@ -1 +1 @@ -Subproject commit 5266f24da75dc449bd56cbed7addb9c8e4a6a73e +Subproject commit 8642541b255bfb5dc39cd351a178be09013a55d6 diff --git a/llama-cpp-bindings-sys/wrapper.h b/llama-cpp-bindings-sys/wrapper.h index e8469f430..d1062c26e 100644 --- a/llama-cpp-bindings-sys/wrapper.h +++ b/llama-cpp-bindings-sys/wrapper.h @@ -1,5 +1,6 @@ #include "llama.cpp/include/llama.h" #include "llama.cpp/ggml/include/gguf.h" +#include "wrapper_batch.h" #include "wrapper_chat_apply.h" #include "wrapper_chat_parse.h" #include "wrapper_common.h" diff --git a/llama-cpp-bindings-sys/wrapper_batch.cpp b/llama-cpp-bindings-sys/wrapper_batch.cpp new file mode 100644 index 000000000..f1de0153c --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_batch.cpp @@ -0,0 +1,111 @@ +#include "wrapper_batch.h" +#include "wrapper_report_current_exception.h" + +#include + +extern "C" auto llama_rs_batch_ext_init( + struct llama_context * ctx, + struct llama_batch_ext ** out_batch, + char ** out_error) -> llama_rs_batch_ext_init_status { + if (out_batch != nullptr) { + *out_batch = nullptr; + } + if (out_error != nullptr) { + *out_error = nullptr; + } + if (ctx == nullptr) { + return LLAMA_RS_BATCH_EXT_INIT_NULL_CTX_ARG; + } + if (out_batch == nullptr) { + return LLAMA_RS_BATCH_EXT_INIT_NULL_OUT_BATCH_ARG; + } + if (out_error == nullptr) { + return LLAMA_RS_BATCH_EXT_INIT_NULL_OUT_ERROR_ARG; + } + + try { + *out_batch = llama_batch_ext_init(ctx); + + return LLAMA_RS_BATCH_EXT_INIT_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_BATCH_EXT_INIT_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_BATCH_EXT_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_BATCH_EXT_INIT_ERROR_STRING_ALLOCATION_FAILED); + } +} + +extern "C" auto llama_rs_batch_ext_add_entry( + struct llama_batch_ext * batch, + llama_token token, + llama_pos pos, + const llama_seq_id * seq_ids, + size_t n_seq_ids, + bool output, + int32_t * out_llama_cpp_return_code, + char ** out_error) -> llama_rs_batch_ext_add_entry_status { + if (out_llama_cpp_return_code != nullptr) { + *out_llama_cpp_return_code = 0; + } + if (out_error != nullptr) { + *out_error = nullptr; + } + if (batch == nullptr) { + return LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_BATCH_ARG; + } + if (seq_ids == nullptr) { + return LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_SEQ_IDS_ARG; + } + if (out_llama_cpp_return_code == nullptr) { + return LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG; + } + if (out_error == nullptr) { + return LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_OUT_ERROR_ARG; + } + if (n_seq_ids == 0) { + return LLAMA_RS_BATCH_EXT_ADD_ENTRY_EMPTY_SEQ_IDS; + } + + try { + const gsl::span seq_id_span(seq_ids, n_seq_ids); + const int32_t index = llama_batch_ext_add_token(batch, seq_id_span[0], token); + + if (index < 0) { + *out_llama_cpp_return_code = index; + + switch (index) { + case -1: + return LLAMA_RS_BATCH_EXT_ADD_ENTRY_BATCH_FULL; + case -2: + return LLAMA_RS_BATCH_EXT_ADD_ENTRY_INVALID_TOKEN; + case -3: + return LLAMA_RS_BATCH_EXT_ADD_ENTRY_INVALID_SEQUENCE_ID; + default: + return LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_RETURNED_UNKNOWN_CODE; + } + } + + for (const llama_seq_id seq_id : seq_id_span.subspan(1)) { + if (!llama_batch_ext_add_seq(batch, index, seq_id)) { + return LLAMA_RS_BATCH_EXT_ADD_ENTRY_INVALID_SEQUENCE_ID; + } + } + + if (!llama_batch_ext_set_pos(batch, index, &pos)) { + return LLAMA_RS_BATCH_EXT_ADD_ENTRY_POSITION_REJECTED; + } + + if (!llama_batch_ext_set_output_logits(batch, index, output)) { + return LLAMA_RS_BATCH_EXT_ADD_ENTRY_OUTPUT_REJECTED; + } + + return LLAMA_RS_BATCH_EXT_ADD_ENTRY_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_BATCH_EXT_ADD_ENTRY_ERROR_STRING_ALLOCATION_FAILED); + } +} diff --git a/llama-cpp-bindings-sys/wrapper_batch.h b/llama-cpp-bindings-sys/wrapper_batch.h new file mode 100644 index 000000000..660fde705 --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_batch.h @@ -0,0 +1,58 @@ +#pragma once + +#include "llama.cpp/include/llama.h" + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum llama_rs_batch_ext_init_status { + LLAMA_RS_BATCH_EXT_INIT_OK = 0, + LLAMA_RS_BATCH_EXT_INIT_NULL_CTX_ARG, + LLAMA_RS_BATCH_EXT_INIT_NULL_OUT_BATCH_ARG, + LLAMA_RS_BATCH_EXT_INIT_NULL_OUT_ERROR_ARG, + LLAMA_RS_BATCH_EXT_INIT_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_BATCH_EXT_INIT_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_BATCH_EXT_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_batch_ext_init_status; + +llama_rs_batch_ext_init_status llama_rs_batch_ext_init( + struct llama_context * ctx, + struct llama_batch_ext ** out_batch, + char ** out_error); + +typedef enum llama_rs_batch_ext_add_entry_status { + LLAMA_RS_BATCH_EXT_ADD_ENTRY_OK = 0, + LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_BATCH_ARG, + LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_SEQ_IDS_ARG, + LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG, + LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_OUT_ERROR_ARG, + LLAMA_RS_BATCH_EXT_ADD_ENTRY_EMPTY_SEQ_IDS, + LLAMA_RS_BATCH_EXT_ADD_ENTRY_BATCH_FULL, + LLAMA_RS_BATCH_EXT_ADD_ENTRY_INVALID_TOKEN, + LLAMA_RS_BATCH_EXT_ADD_ENTRY_INVALID_SEQUENCE_ID, + LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_RETURNED_UNKNOWN_CODE, + LLAMA_RS_BATCH_EXT_ADD_ENTRY_POSITION_REJECTED, + LLAMA_RS_BATCH_EXT_ADD_ENTRY_OUTPUT_REJECTED, + LLAMA_RS_BATCH_EXT_ADD_ENTRY_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_batch_ext_add_entry_status; + +llama_rs_batch_ext_add_entry_status llama_rs_batch_ext_add_entry( + struct llama_batch_ext * batch, + llama_token token, + llama_pos pos, + const llama_seq_id * seq_ids, + size_t n_seq_ids, + bool output, + int32_t * out_llama_cpp_return_code, + char ** out_error); + +#ifdef __cplusplus +} +#endif diff --git a/llama-cpp-bindings-sys/wrapper_chat_apply.cpp b/llama-cpp-bindings-sys/wrapper_chat_apply.cpp index 35618c217..505e711f6 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_apply.cpp +++ b/llama-cpp-bindings-sys/wrapper_chat_apply.cpp @@ -74,7 +74,7 @@ extern "C" auto llama_rs_apply_chat_template( inputs.add_generation_prompt = add_generation_prompt != 0; inputs.enable_thinking = enable_thinking != 0; - std::string const rendered = common_chat_template_direct_apply(tmpl, inputs); + std::string const rendered = common_chat_template_direct_apply(tmpl, inputs).str(); if (rendered.empty()) { return LLAMA_RS_APPLY_CHAT_TEMPLATE_TEMPLATE_APPLICATION_FAILED; } diff --git a/llama-cpp-bindings-sys/wrapper_chat_parse.cpp b/llama-cpp-bindings-sys/wrapper_chat_parse.cpp index b90302f7a..fe661ac2d 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_parse.cpp +++ b/llama-cpp-bindings-sys/wrapper_chat_parse.cpp @@ -45,7 +45,7 @@ auto parse_with_tools_parser( parser_params.format = COMMON_CHAT_FORMAT_PEG_NATIVE; common_chat_msg parsed = - common_chat_peg_parse(chat_parser, input, is_partial != 0, parser_params); + common_chat_peg_parse(chat_parser, common_chat_input(std::string(input)), is_partial != 0, parser_params); auto handle = std::make_unique(); handle->message = std::move(parsed); diff --git a/llama-cpp-bindings-sys/wrapper_common.cpp b/llama-cpp-bindings-sys/wrapper_common.cpp index cf653e7da..ada517d0a 100644 --- a/llama-cpp-bindings-sys/wrapper_common.cpp +++ b/llama-cpp-bindings-sys/wrapper_common.cpp @@ -16,7 +16,6 @@ extern "C" auto llama_rs_json_schema_to_grammar( const char * schema_json, - bool force_gbnf, char ** out_grammar, char ** out_error) -> llama_rs_json_schema_to_grammar_status { if (out_grammar != nullptr) { @@ -37,12 +36,18 @@ extern "C" auto llama_rs_json_schema_to_grammar( try { const auto schema = common_json::parse(schema_json); - const auto grammar = json_schema_to_grammar(schema, force_gbnf); + const auto grammar = json_schema_to_grammar(schema); *out_grammar = llama_rs_dup_string(grammar); if (*out_grammar == nullptr) { return LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_ERROR_STRING_ALLOCATION_FAILED; } return LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_OK; + } catch (const common_json_error & err) { + return llama_rs_report_exception_message( + out_error, + err.what(), + LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_INVALID_JSON, + LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_ERROR_STRING_ALLOCATION_FAILED); } catch (const std::invalid_argument & err) { return llama_rs_report_exception_message( out_error, @@ -195,7 +200,7 @@ extern "C" auto llama_rs_memory_seq_pos_max( extern "C" auto llama_rs_encode( struct llama_context * ctx, - struct llama_batch batch, + struct llama_batch_ext * batch, int32_t * out_llama_cpp_return_code, char ** out_error) -> llama_rs_encode_status { if (out_error != nullptr) { @@ -207,6 +212,9 @@ extern "C" auto llama_rs_encode( if (ctx == nullptr) { return LLAMA_RS_ENCODE_NULL_CTX_ARG; } + if (batch == nullptr) { + return LLAMA_RS_ENCODE_NULL_BATCH_ARG; + } try { const auto * model = llama_get_model(ctx); if (model == nullptr) { @@ -215,7 +223,7 @@ extern "C" auto llama_rs_encode( if (!llama_model_has_encoder(model)) { return LLAMA_RS_ENCODE_MODEL_HAS_NO_ENCODER; } - int32_t const result = llama_encode(ctx, batch); + int32_t const result = llama_process(ctx, LLAMA_PROCESS_TYPE_ENCODE, batch); if (result != 0) { if (out_llama_cpp_return_code != nullptr) { *out_llama_cpp_return_code = result; @@ -442,7 +450,7 @@ extern "C" auto llama_rs_new_context_with_model( extern "C" auto llama_rs_decode( struct llama_context * ctx, - struct llama_batch batch, + struct llama_batch_ext * batch, int32_t * out_llama_cpp_return_code, char ** out_error) -> llama_rs_decode_status { if (out_error != nullptr) { @@ -454,11 +462,14 @@ extern "C" auto llama_rs_decode( if (ctx == nullptr) { return LLAMA_RS_DECODE_NULL_CTX_ARG; } + if (batch == nullptr) { + return LLAMA_RS_DECODE_NULL_BATCH_ARG; + } if (out_error == nullptr) { return LLAMA_RS_DECODE_NULL_OUT_ERROR_ARG; } try { - int32_t const result = llama_decode(ctx, batch); + int32_t const result = llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch); if (result != 0) { if (out_llama_cpp_return_code != nullptr) { *out_llama_cpp_return_code = result; diff --git a/llama-cpp-bindings-sys/wrapper_common.h b/llama-cpp-bindings-sys/wrapper_common.h index b9a7330d1..f99bf9056 100644 --- a/llama-cpp-bindings-sys/wrapper_common.h +++ b/llama-cpp-bindings-sys/wrapper_common.h @@ -24,12 +24,12 @@ typedef enum llama_rs_json_schema_to_grammar_status { LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_ERROR_STRING_ALLOCATION_FAILED, LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_LLAMA_CPP_OUT_OF_MEMORY, LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_INVALID_SCHEMA, + LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_INVALID_JSON, LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_LLAMA_CPP_THREW_CXX_EXCEPTION, } llama_rs_json_schema_to_grammar_status; llama_rs_json_schema_to_grammar_status llama_rs_json_schema_to_grammar( const char * schema_json, - bool force_gbnf, char ** out_grammar, char ** out_error); @@ -129,6 +129,7 @@ llama_rs_memory_seq_pos_max_status llama_rs_memory_seq_pos_max( typedef enum llama_rs_encode_status { LLAMA_RS_ENCODE_OK = 0, LLAMA_RS_ENCODE_NULL_CTX_ARG, + LLAMA_RS_ENCODE_NULL_BATCH_ARG, LLAMA_RS_ENCODE_NULL_MODEL, LLAMA_RS_ENCODE_MODEL_HAS_NO_ENCODER, LLAMA_RS_ENCODE_LLAMA_CPP_RETURNED_NONZERO_CODE, @@ -141,7 +142,7 @@ typedef enum llama_rs_encode_status { llama_rs_encode_status llama_rs_encode( struct llama_context * ctx, - struct llama_batch batch, + struct llama_batch_ext * batch, int32_t * out_llama_cpp_return_code, char ** out_error); @@ -220,6 +221,7 @@ llama_rs_new_context_with_model_status llama_rs_new_context_with_model( typedef enum llama_rs_decode_status { LLAMA_RS_DECODE_OK = 0, LLAMA_RS_DECODE_NULL_CTX_ARG, + LLAMA_RS_DECODE_NULL_BATCH_ARG, LLAMA_RS_DECODE_NULL_OUT_ERROR_ARG, LLAMA_RS_DECODE_LLAMA_CPP_RETURNED_NONZERO_CODE, LLAMA_RS_DECODE_OUT_OF_MEMORY, @@ -231,7 +233,7 @@ typedef enum llama_rs_decode_status { llama_rs_decode_status llama_rs_decode( struct llama_context * ctx, - struct llama_batch batch, + struct llama_batch_ext * batch, int32_t * out_llama_cpp_return_code, char ** out_error); diff --git a/llama-cpp-bindings-sys/wrapper_context.cpp b/llama-cpp-bindings-sys/wrapper_context.cpp index 4b06b775a..825e6a39e 100644 --- a/llama-cpp-bindings-sys/wrapper_context.cpp +++ b/llama-cpp-bindings-sys/wrapper_context.cpp @@ -9,10 +9,6 @@ #include "llama.cpp/src/llama-model.h" #include "wrapper_report_current_exception.h" -extern "C" auto llama_rs_context_decodes_batches_in_one_micro_batch(const struct llama_context * ctx) -> bool { - return llama_get_memory(ctx) == nullptr || !ctx->get_cparams().causal_attn; -} - extern "C" auto llama_rs_context_embeddings_enabled(const struct llama_context * ctx) -> bool { return ctx->get_cparams().embeddings; } diff --git a/llama-cpp-bindings-sys/wrapper_context.h b/llama-cpp-bindings-sys/wrapper_context.h index c8d58b28c..a382190d9 100644 --- a/llama-cpp-bindings-sys/wrapper_context.h +++ b/llama-cpp-bindings-sys/wrapper_context.h @@ -21,8 +21,6 @@ typedef enum llama_rs_context_nextn_hidden_state_probe_status { LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_THREW_CXX_EXCEPTION, } llama_rs_context_nextn_hidden_state_probe_status; -bool llama_rs_context_decodes_batches_in_one_micro_batch(const struct llama_context * ctx); - bool llama_rs_context_embeddings_enabled(const struct llama_context * ctx); bool llama_rs_context_embeddings_nextn_enabled(const struct llama_context * ctx); diff --git a/llama-cpp-bindings-sys/wrapper_gbnf.cpp b/llama-cpp-bindings-sys/wrapper_gbnf.cpp index 2cff7b500..5fc9f4e90 100644 --- a/llama-cpp-bindings-sys/wrapper_gbnf.cpp +++ b/llama-cpp-bindings-sys/wrapper_gbnf.cpp @@ -40,7 +40,7 @@ extern "C" auto llama_rs_validate_gbnf( vocab, grammar_str, grammar_root, false, nullptr, 0, nullptr, 0); if (grammar == nullptr) { - return LLAMA_RS_GBNF_VALIDATION_LEFT_RECURSION; + return LLAMA_RS_GBNF_VALIDATION_UNSUPPORTED_GRAMMAR; } llama_grammar_free_impl(grammar); diff --git a/llama-cpp-bindings-sys/wrapper_gbnf.h b/llama-cpp-bindings-sys/wrapper_gbnf.h index 337167524..8ca391ef9 100644 --- a/llama-cpp-bindings-sys/wrapper_gbnf.h +++ b/llama-cpp-bindings-sys/wrapper_gbnf.h @@ -11,7 +11,7 @@ typedef enum llama_rs_gbnf_validation_status { LLAMA_RS_GBNF_VALIDATION_SYNTAX_ERROR, LLAMA_RS_GBNF_VALIDATION_EMPTY_RULE_SET, LLAMA_RS_GBNF_VALIDATION_ROOT_SYMBOL_MISSING, - LLAMA_RS_GBNF_VALIDATION_LEFT_RECURSION, + LLAMA_RS_GBNF_VALIDATION_UNSUPPORTED_GRAMMAR, LLAMA_RS_GBNF_VALIDATION_NULL_GRAMMAR_ARG, LLAMA_RS_GBNF_VALIDATION_NULL_ROOT_ARG, LLAMA_RS_GBNF_VALIDATION_NULL_OUT_ERROR_ARG, diff --git a/llama-cpp-bindings-sys/wrapper_reasoning.cpp b/llama-cpp-bindings-sys/wrapper_reasoning.cpp index 327b673e5..ba0cb4f1f 100644 --- a/llama-cpp-bindings-sys/wrapper_reasoning.cpp +++ b/llama-cpp-bindings-sys/wrapper_reasoning.cpp @@ -24,7 +24,6 @@ namespace { auto find_reasoning_markers( const common_chat_template & tmpl, - const char * tmpl_src, std::string * out_start, std::vector * out_ends) -> bool { autoparser::generation_params probe_params; @@ -37,8 +36,7 @@ auto find_reasoning_markers( common_json{ { "role", "user" }, { "content", "ping" } }, }); - const std::string tmpl_src_str = tmpl_src; - if (auto specialized = common_chat_try_specialized_template(tmpl, tmpl_src_str, probe_params)) { + if (auto specialized = common_chat_try_specialized_template(tmpl, probe_params)) { if (specialized->supports_thinking && !specialized->thinking_start_tag.empty() && !specialized->thinking_end_tags.empty()) { @@ -100,7 +98,7 @@ extern "C" auto llama_rs_detect_reasoning_markers( common_chat_template const tmpl(tmpl_src, bos_token, eos_token); auto detected = std::make_unique(); - if (!find_reasoning_markers(tmpl, tmpl_src, &detected->open, &detected->closes)) { + if (!find_reasoning_markers(tmpl, &detected->open, &detected->closes)) { return LLAMA_RS_DETECT_REASONING_MARKERS_OK; } diff --git a/llama-cpp-bindings-tests/src/classify_sample_loop.rs b/llama-cpp-bindings-tests/src/classify_sample_loop.rs index 941eed4db..d5949b182 100644 --- a/llama-cpp-bindings-tests/src/classify_sample_loop.rs +++ b/llama-cpp-bindings-tests/src/classify_sample_loop.rs @@ -8,11 +8,11 @@ use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; -pub struct ClassifySampleLoop<'borrow, 'model, 'tokens> { +pub struct ClassifySampleLoop<'borrow, 'model> { pub classifier: &'borrow mut SampledTokenClassifier<'model>, pub sampler: &'borrow mut LlamaSampler, pub context: &'borrow mut LlamaContext<'model>, - pub batch: &'borrow mut LlamaBatch<'tokens>, + pub batch: &'borrow mut LlamaBatch, pub initial_position: i32, pub max_generated_tokens: i32, } @@ -59,7 +59,7 @@ impl ClassifySampleLoopOutcome { } } -impl ClassifySampleLoop<'_, '_, '_> { +impl ClassifySampleLoop<'_, '_> { /// # Errors /// Forwards [`SampledTokenClassifier::sample`] / [`LlamaContext::decode`] / /// [`LlamaBatch::add`] errors verbatim. Stops on the end of generation, on diff --git a/llama-cpp-bindings-tests/src/prime_kv_cache.rs b/llama-cpp-bindings-tests/src/prime_kv_cache.rs index 1123abd30..35fd864a5 100644 --- a/llama-cpp-bindings-tests/src/prime_kv_cache.rs +++ b/llama-cpp-bindings-tests/src/prime_kv_cache.rs @@ -1,8 +1,5 @@ -use std::num::NonZeroU16; - use anyhow::Result; use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_bindings_types::BatchCapacity; use llama_cpp_test_harness::LlamaFixture; use crate::prime_kv_cache_with::prime_kv_cache_with; @@ -10,10 +7,5 @@ use crate::prime_kv_cache_with::prime_kv_cache_with; /// # Errors /// Forwards tokenization and [`LlamaContext::decode`] errors verbatim. pub fn prime_kv_cache(fixture: &LlamaFixture<'_>, context: &mut LlamaContext<'_>) -> Result<()> { - prime_kv_cache_with( - fixture, - context, - "Hello world", - BatchCapacity::from(NonZeroU16::MAX), - ) + prime_kv_cache_with(fixture, context, "Hello world") } diff --git a/llama-cpp-bindings-tests/src/prime_kv_cache_with.rs b/llama-cpp-bindings-tests/src/prime_kv_cache_with.rs index 8f8a9ba78..38b7ec2e4 100644 --- a/llama-cpp-bindings-tests/src/prime_kv_cache_with.rs +++ b/llama-cpp-bindings-tests/src/prime_kv_cache_with.rs @@ -1,9 +1,9 @@ use anyhow::Result; use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::AddBos; use llama_cpp_bindings::model::ParseSpecialTokens; -use llama_cpp_bindings_types::BatchCapacity; use llama_cpp_test_harness::LlamaFixture; /// # Errors @@ -12,13 +12,12 @@ pub fn prime_kv_cache_with( fixture: &LlamaFixture<'_>, context: &mut LlamaContext<'_>, text: &str, - batch_capacity: BatchCapacity, ) -> Result<()> { let tokens = fixture .model .str_to_token(text, AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(batch_capacity, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/batch_context_limits.rs b/llama-cpp-bindings-tests/tests/batch_context_limits.rs new file mode 100644 index 000000000..04a87a9f2 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/batch_context_limits.rs @@ -0,0 +1,42 @@ +use anyhow::Result; +use llama_cpp_bindings::DecodeError; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::error::batch_transfer_error::BatchTransferError; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn decoding_a_batch_built_for_other_context_limits_is_refused( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + let context_limits = ContextLimits::of(&context); + let other_limits = ContextLimits { + n_batch: context_limits.n_batch * 2, + ..context_limits + }; + let mut batch = LlamaBatch::new(other_limits); + batch.add(&SampledToken::Content(LlamaToken::new(1)), 0, &[0], true)?; + + assert_eq!( + context.decode(&batch), + Err(DecodeError::BatchTransfer( + BatchTransferError::LimitsMismatch { + batch: other_limits, + context: context_limits, + } + )) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/context_after_decoding_text.rs b/llama-cpp-bindings-tests/tests/context_after_decoding_text.rs index d5c63d40e..ea559dfee 100644 --- a/llama-cpp-bindings-tests/tests/context_after_decoding_text.rs +++ b/llama-cpp-bindings-tests/tests/context_after_decoding_text.rs @@ -1,10 +1,10 @@ use anyhow::Result; use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::context::params::LlamaPoolingType; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::AddBos; use llama_cpp_bindings::model::ParseSpecialTokens; -use llama_cpp_bindings_types::BatchCapacity; use llama_cpp_test_harness::LlamaFixture; const TEXT: &str = "The parcel arrived on the last working day of the week."; @@ -23,10 +23,10 @@ pub fn context_after_decoding_text<'fixture>( let tokens = fixture .model .str_to_token(TEXT, AddBos::Always, ParseSpecialTokens::Never)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; Ok(context) } diff --git a/llama-cpp-bindings-tests/tests/context_state.rs b/llama-cpp-bindings-tests/tests/context_state.rs index 110738792..1e498342c 100644 --- a/llama-cpp-bindings-tests/tests/context_state.rs +++ b/llama-cpp-bindings-tests/tests/context_state.rs @@ -9,7 +9,10 @@ use llama_cpp_bindings::DecodeError; use llama_cpp_bindings::EmbeddingsError; use llama_cpp_bindings::LlamaContextLoadError; use llama_cpp_bindings::LogitsError; +use llama_cpp_bindings::StringToTokenError; +use llama_cpp_bindings::batch_add_error::BatchAddError; use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::context::params::LlamaAttentionType; use llama_cpp_bindings::context::params::LlamaPoolingType; use llama_cpp_bindings::error::ClearKvCacheSeqError; @@ -25,7 +28,6 @@ use llama_cpp_bindings::token::data::LlamaTokenData; use llama_cpp_bindings::token::data_array::LlamaTokenDataArray; use llama_cpp_bindings_tests::prime_kv_cache::prime_kv_cache; use llama_cpp_bindings_tests::prime_kv_cache_with::prime_kv_cache_with; -use llama_cpp_bindings_types::BatchCapacity; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::download_model::download_model; use llama_cpp_test_harness::llama_test; @@ -228,10 +230,10 @@ fn decode_and_get_logits(fixture: &LlamaFixture<'_>) -> Result<()> { let tokens = fixture .model .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - let decode_result = context.decode(&mut batch); + let decode_result = context.decode(&batch); assert!(decode_result.is_ok()); let logits = context.get_logits()?; @@ -279,9 +281,9 @@ fn token_data_array_has_entries_after_decode(fixture: &LlamaFixture<'_>) -> Resu let tokens = fixture .model .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let token_data_array = context.token_data_array()?; @@ -308,9 +310,9 @@ fn get_logits_ith_returns_valid_slice(fixture: &LlamaFixture<'_>) -> Result<()> .model .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; let last_index = i32::try_from(tokens.len() - 1)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let logits = context.get_logits_ith(last_index)?; @@ -337,9 +339,9 @@ fn token_data_array_ith_returns_valid_data(fixture: &LlamaFixture<'_>) -> Result .model .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; let last_index = i32::try_from(tokens.len() - 1)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let token_data_array = context.token_data_array_ith(last_index)?; @@ -413,9 +415,9 @@ fn candidates_returns_n_vocab_entries(fixture: &LlamaFixture<'_>) -> Result<()> let tokens = fixture .model .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let count = context.candidates()?.count(); @@ -463,9 +465,9 @@ fn candidates_ith_returns_n_vocab_entries(fixture: &LlamaFixture<'_>) -> Result< .model .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; let last_index = i32::try_from(tokens.len() - 1)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let count = context.candidates_ith(last_index)?.count(); @@ -548,10 +550,10 @@ fn encode_on_non_encoder_model_returns_error(fixture: &LlamaFixture<'_>) -> Resu let tokens = fixture .model .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - let result = context.encode(&mut batch); + let result = context.encode(&batch); assert!(result.is_err()); @@ -580,9 +582,9 @@ fn embeddings_seq_ith_returns_null_embedding_error_for_invalid_seq( let tokens = fixture .model .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; assert_eq!( context.embeddings_seq_ith(999), @@ -608,9 +610,9 @@ fn decode_empty_batch_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { fixture.backend, (*fixture.context_params).into_llama_context_params(), )?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let batch = LlamaBatch::new(ContextLimits::of(&context)); - let result = context.decode(&mut batch); + let result = context.decode(&batch); assert!(result.is_err()); @@ -637,10 +639,10 @@ fn set_abort_flag_aborts_decode(fixture: &LlamaFixture<'_>) -> Result<()> { let tokens = fixture .model .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - let result = context.decode(&mut batch); + let result = context.decode(&batch); assert_eq!(result, Err(DecodeError::Aborted)); @@ -667,10 +669,10 @@ fn set_abort_flag_false_allows_decode(fixture: &LlamaFixture<'_>) -> Result<()> let tokens = fixture .model .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - let result = context.decode(&mut batch); + let result = context.decode(&batch); assert!(result.is_ok()); @@ -698,10 +700,10 @@ fn clear_abort_callback_allows_decode_with_flag_true(fixture: &LlamaFixture<'_>) let tokens = fixture .model .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - let result = context.decode(&mut batch); + let result = context.decode(&batch); assert!(result.is_ok()); @@ -772,38 +774,6 @@ fn get_logits_ith_returns_token_not_initialized_for_unknown_index( Ok(()) } -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 64, - n_batch = 2048, - n_ubatch = 512, -)] -fn get_logits_ith_returns_token_index_exceeds_context_for_huge_index( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let huge_index = i32::try_from(context.n_ctx())?; - context.mark_logits_initialized(huge_index); - let result = context.get_logits_ith(huge_index); - - assert_eq!( - result, - Err(LogitsError::TokenIndexExceedsContext { - token_index: u32::try_from(huge_index)?, - context_size: context.n_ctx(), - }) - ); - - Ok(()) -} - #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -901,33 +871,23 @@ fn prime_kv_cache_surfaces_each_underlying_error(fixture: &LlamaFixture<'_>) -> let mut context = fixture.build_context()?; assert!( - prime_kv_cache_with( - fixture, - &mut context, - "Hello\0world", - BatchCapacity::try_from(512)? - ) - .is_err(), + prime_kv_cache_with(fixture, &mut context, "Hello\0world") + .is_err_and(|error| error.downcast_ref::().is_some()), "an interior null byte must surface a tokenization error" ); - assert!( - prime_kv_cache_with( - fixture, - &mut context, - &"word ".repeat(64), - BatchCapacity::try_from(4)? - ) - .is_err(), - "more tokens than the batch capacity must surface an add-sequence error" + assert_eq!( + prime_kv_cache_with(fixture, &mut context, &"word ".repeat(300)) + .err() + .and_then(|error| error.downcast_ref::().cloned()), + Some(BatchAddError::BatchFull { n_batch: 256 }), + "more tokens than one decode takes must surface an add-sequence error" ); let filler = "word ".repeat(40); - let filler_batch_capacity = BatchCapacity::try_from(256)?; - let mut decode_result = - prime_kv_cache_with(fixture, &mut context, &filler, filler_batch_capacity); + let mut decode_result = prime_kv_cache_with(fixture, &mut context, &filler); let mut attempts = 0; while decode_result.is_ok() && attempts < 16 { - decode_result = prime_kv_cache_with(fixture, &mut context, &filler, filler_batch_capacity); + decode_result = prime_kv_cache_with(fixture, &mut context, &filler); attempts += 1; } assert!( @@ -1685,9 +1645,9 @@ fn save_and_load_session_file(fixture: &LlamaFixture<'_>) -> Result<()> { fixture .model .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let session_path = std::env::temp_dir().join("llama_test_session.bin"); context.state_save_file(&session_path, &tokens)?; @@ -1779,9 +1739,9 @@ fn state_seq_save_and_load_file_roundtrip(fixture: &LlamaFixture<'_>) -> Result< fixture .model .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let session_path = std::env::temp_dir().join("llama_test_seq_state.bin"); let bytes_written = context.state_seq_save_file(&session_path, 0, &tokens)?; @@ -1839,9 +1799,9 @@ fn set_state_data_rejects_a_truncated_snapshot(fixture: &LlamaFixture<'_>) -> Re fixture .model .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let mut state_data = vec![0u8; context.get_state_size()]; let bytes_copied = unsafe { context.copy_state_data(&mut state_data) }?; @@ -1875,9 +1835,9 @@ fn copy_state_data_and_set_state_data_roundtrip(fixture: &LlamaFixture<'_>) -> R fixture .model .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let state_size = context.get_state_size(); let mut state_data = vec![0u8; state_size]; @@ -2106,9 +2066,9 @@ fn state_load_file_with_zero_max_tokens_returns_error(fixture: &LlamaFixture<'_> fixture .model .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let session_path = std::env::temp_dir().join("llama_test_session_zero_max.bin"); context.state_save_file(&session_path, &tokens)?; @@ -2162,9 +2122,9 @@ fn state_seq_load_file_with_zero_max_tokens_returns_error( fixture .model .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let session_path = std::env::temp_dir().join("llama_test_seq_state_zero_max.bin"); context.state_seq_save_file(&session_path, 0, &tokens)?; @@ -2219,9 +2179,9 @@ fn state_load_file_with_insufficient_max_tokens_returns_length_error( AddBos::Always, ParseSpecialTokens::Always, )?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let session_path = std::env::temp_dir().join("llama_test_session_insuf.bin"); context.state_save_file(&session_path, &tokens)?; @@ -2276,9 +2236,9 @@ fn state_seq_load_file_with_insufficient_max_tokens_returns_length_error( AddBos::Always, ParseSpecialTokens::Always, )?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let session_path = std::env::temp_dir().join("llama_test_seq_state_insuf.bin"); context.state_seq_save_file(&session_path, 0, &tokens)?; @@ -2694,9 +2654,9 @@ fn state_seq_get_size_ext_returns_size_for_decoded_sequence( fixture .model .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let flags = LlamaStateSeqFlags::empty(); let size = context.state_seq_get_size_ext(0, &flags); @@ -2747,9 +2707,9 @@ fn state_seq_get_data_ext_and_set_data_ext_round_trip(fixture: &LlamaFixture<'_> fixture .model .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let flags = LlamaStateSeqFlags::empty(); let size = context.state_seq_get_size_ext(0, &flags); @@ -2786,10 +2746,10 @@ fn qwen35_refilled_candidate_array_matches_a_freshly_built_one( (*fixture.context_params).into_llama_context_params(), )?; let prompt_tokens = model.str_to_token("Hello", AddBos::Never, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(64)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&prompt_tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let last_index = batch.n_tokens() - 1; let mut reused_candidates = LlamaTokenDataArray::new( @@ -2828,12 +2788,12 @@ fn decoding_more_tokens_than_the_micro_batch_with_non_causal_attention_returns_a AddBos::Always, ParseSpecialTokens::Always, )?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; let n_tokens = batch.n_tokens(); assert_eq!( - context.decode(&mut batch), + context.decode(&batch), Err(DecodeError::BatchExceedsMicroBatch { n_tokens, n_ubatch: context.n_ubatch(), diff --git a/llama-cpp-bindings-tests/tests/embedding_models.rs b/llama-cpp-bindings-tests/tests/embedding_models.rs index bbaf72c7c..34125e3e2 100644 --- a/llama-cpp-bindings-tests/tests/embedding_models.rs +++ b/llama-cpp-bindings-tests/tests/embedding_models.rs @@ -15,6 +15,7 @@ use llama_cpp_bindings::KvCacheSeqAddError; use llama_cpp_bindings::KvCacheSeqDivError; use llama_cpp_bindings::KvCacheSeqPosMaxError; use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::context::params::LlamaPoolingType; use llama_cpp_bindings::ggml_time_us; use llama_cpp_bindings::gguf_context::GgufContext; @@ -25,7 +26,6 @@ use llama_cpp_bindings::model::ParseSpecialTokens; use llama_cpp_bindings::model::params::LlamaModelParams; use llama_cpp_bindings::token::LlamaToken; use llama_cpp_bindings_tests::prime_kv_cache::prime_kv_cache; -use llama_cpp_bindings_types::BatchCapacity; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; use toktrie::INVALID_TOKEN; @@ -80,14 +80,14 @@ fn embedding_generation_produces_vectors(fixture: &LlamaFixture<'_>) -> Result<( let t_main_start = ggml_time_us(); let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(ctx.n_ctx())?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; assert_eq!(classifier.pending_prompt_tokens(), prompt_token_count); assert_eq!(classifier.usage().prompt_tokens, 0); ctx.clear_kv_cache(); - ctx.decode(&mut batch) + ctx.decode(&batch) .with_context(|| "llama_decode() failed")?; let promoted = classifier.commit_prompt_tokens(); @@ -181,10 +181,7 @@ fn reranking_produces_scores(fixture: &LlamaFixture<'_>) -> Result<()> { } let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut batch = LlamaBatch::new( - BatchCapacity::try_from(2048)?, - i32::try_from(document_count)?, - ); + let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); let t_main_start = ggml_time_us(); for (sequence_index, tokens) in tokens_lines_list.iter().enumerate() { @@ -203,7 +200,7 @@ fn reranking_produces_scores(fixture: &LlamaFixture<'_>) -> Result<()> { assert_eq!(classifier.usage().prompt_tokens, 0); ctx.clear_kv_cache(); - ctx.decode(&mut batch) + ctx.decode(&batch) .with_context(|| "llama_decode() failed")?; let promoted = classifier.commit_prompt_tokens(); @@ -274,10 +271,10 @@ fn decode_with_embeddings_enabled(fixture: &LlamaFixture<'_>) -> Result<()> { let tokens = fixture .model .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - let result = context.decode(&mut batch); + let result = context.decode(&batch); assert!(result.is_ok()); @@ -302,9 +299,9 @@ fn embeddings_seq_ith_returns_valid_embeddings(fixture: &LlamaFixture<'_>) -> Re let tokens = fixture .model .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let embeddings = context.embeddings_seq_ith(0)?; @@ -338,7 +335,7 @@ fn multi_sequence_embeddings_returns_one_embedding_per_sequence( "gamma waits", "delta jumps", ]; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(64)?, 4); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); for (sequence_index, text) in inputs.iter().enumerate() { let tokens = @@ -350,7 +347,7 @@ fn multi_sequence_embeddings_returns_one_embedding_per_sequence( batch.add_sequence(&tokens, sequence_id, true)?; } - context.decode(&mut batch)?; + context.decode(&batch)?; let n_embd = usize::try_from(fixture.model.n_embd())?; let mut collected: Vec> = Vec::with_capacity(inputs.len()); @@ -411,7 +408,7 @@ fn embeddings_returns_distinct_values_when_reused_batch_has_extra_capacity( ]; let n_embd = usize::try_from(fixture.model.n_embd())?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(64)?, 4); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); let mut collected: Vec> = Vec::new(); for iteration_inputs in iterations { @@ -426,7 +423,7 @@ fn embeddings_returns_distinct_values_when_reused_batch_has_extra_capacity( } context.clear_kv_cache(); - context.decode(&mut batch)?; + context.decode(&batch)?; for sequence_index in 0..iteration_inputs.len() { let sequence_id = i32::try_from(sequence_index)?; @@ -483,9 +480,9 @@ fn embeddings_ith_returns_valid_embeddings(fixture: &LlamaFixture<'_>) -> Result .model .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; let last_index = i32::try_from(tokens.len() - 1)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let embeddings = context.embeddings_ith(last_index)?; @@ -540,10 +537,10 @@ fn encode_succeeds_with_encoder_model(fixture: &LlamaFixture<'_>) -> Result<()> let tokens = fixture .model .str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - let result = context.encode(&mut batch); + let result = context.encode(&batch); assert!(result.is_ok()); @@ -784,12 +781,12 @@ fn decoding_more_tokens_than_the_micro_batch_on_an_encoder_returns_an_error( AddBos::Always, ParseSpecialTokens::Always, )?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; let n_tokens = batch.n_tokens(); assert_eq!( - context.decode(&mut batch), + context.decode(&batch), Err(DecodeError::BatchExceedsMicroBatch { n_tokens, n_ubatch: context.n_ubatch(), @@ -817,12 +814,12 @@ fn encoding_more_tokens_than_the_micro_batch_returns_an_error( AddBos::Never, ParseSpecialTokens::Always, )?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; let n_tokens = batch.n_tokens(); assert_eq!( - context.encode(&mut batch), + context.encode(&batch), Err(EncodeError::BatchExceedsMicroBatch { n_tokens, n_ubatch: context.n_ubatch(), diff --git a/llama-cpp-bindings-tests/tests/generation_control.rs b/llama-cpp-bindings-tests/tests/generation_control.rs index 6c472f9f7..84133d889 100644 --- a/llama-cpp-bindings-tests/tests/generation_control.rs +++ b/llama-cpp-bindings-tests/tests/generation_control.rs @@ -1,4 +1,5 @@ use std::ffi::CStr; +use std::ffi::CString; use std::io::Write; use std::time::Duration; @@ -12,7 +13,9 @@ use llama_cpp_bindings::GenerationProgress; use llama_cpp_bindings::GrammarError; use llama_cpp_bindings::SampledToken; use llama_cpp_bindings::SamplerAcceptError; +use llama_cpp_bindings::SamplingError; use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::ggml_time_us; use llama_cpp_bindings::json_schema_to_grammar; use llama_cpp_bindings::llama_batch::LlamaBatch; @@ -24,7 +27,6 @@ use llama_cpp_bindings::sampled_token_section::SampledTokenSection; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings::token::LlamaToken; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; -use llama_cpp_bindings_types::BatchCapacity; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -69,11 +71,11 @@ fn sample_returns_result_and_succeeds_with_valid_index(fixture: &LlamaFixture<'_ )?; let tokens = model.str_to_token("Hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let mut sampler = LlamaSampler::chain_simple([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; @@ -126,11 +128,11 @@ fn grammar_sampler_constrains_output_to_yes_or_no(fixture: &LlamaFixture<'_>) -> let prompt = "<|im_start|>user\nIs the sky blue? Answer yes or no.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let mut sampler = LlamaSampler::chain_simple([ LlamaSampler::grammar(model, r"root ::= [Yy] [Ee] [Ss] | [Nn] [Oo]", "root")?, @@ -220,11 +222,11 @@ fn json_schema_grammar_sampler_constrains_output_to_json(fixture: &LlamaFixture< let prompt = "<|im_start|>user\nWhat is 2+2? Respond with a JSON object.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let grammar_str = json_schema_to_grammar( r#"{"type": "object", "properties": {"answer": {"type": "string"}}, "required": ["answer"]}"#, @@ -313,12 +315,12 @@ fn sample_with_grammar_produces_constrained_output_in_loop( let prompt = "<|im_start|>user\nIs the sky blue? yes or no<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; classifier.commit_prompt_tokens(); let mut sampler = LlamaSampler::chain_simple([ @@ -404,11 +406,11 @@ fn sample_without_grammar_produces_multiple_tokens(fixture: &LlamaFixture<'_>) - let prompt = "<|im_start|>user\nSay hello<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let mut sampler = LlamaSampler::chain_simple([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; @@ -431,7 +433,7 @@ fn sample_without_grammar_produces_multiple_tokens(fixture: &LlamaFixture<'_>) - batch.clear(); batch.add(&raw_as_sampled, position, &[0], true)?; - context.decode(&mut batch)?; + context.decode(&batch)?; } classifier.finish(&mut outcomes); @@ -479,7 +481,13 @@ fn dry_sampler_with_null_byte_in_seq_breakers_returns_error( let breakers: Vec<&[u8]> = vec![b"hello\0world"]; let result = LlamaSampler::dry(fixture.model, 1.5, 2.0, 128, 2, breakers); - assert!(result.is_err()); + assert_eq!( + result.err(), + Some(SamplingError::DrySequenceBreakerContainsNul { + index: 0, + source: CString::new(&b"hello\0world"[..]).unwrap_err(), + }) + ); Ok(()) } @@ -554,7 +562,7 @@ fn grammar_lazy_with_null_byte_in_pattern_returns_error(fixture: &LlamaFixture<' assert_eq!( result.err(), Some(GrammarError::TriggerPatternContainsNul( - std::ffi::CString::new("hel\0lo").expect_err("the pattern carries a nul byte") + CString::new("hel\0lo").expect_err("the pattern carries a nul byte") )) ); @@ -613,8 +621,7 @@ fn grammar_lazy_with_null_byte_in_grammar_returns_error(fixture: &LlamaFixture<' assert_eq!( result.err(), Some(GrammarError::GrammarContainsNul( - std::ffi::CString::new("root ::= \"hel\0lo\"") - .expect_err("the grammar carries a nul byte") + CString::new("root ::= \"hel\0lo\"").expect_err("the grammar carries a nul byte") )) ); @@ -799,9 +806,9 @@ fn apply_runs_sampler_over_token_data_array(fixture: &LlamaFixture<'_>) -> Resul let tokens = fixture .model .str_to_token("Hi", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let mut data_array = context.token_data_array_ith(batch.n_tokens() - 1)?; let sampler = LlamaSampler::greedy()?; @@ -827,9 +834,9 @@ fn sample_returns_token_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { let tokens = fixture .model .str_to_token("Hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let mut sampler = LlamaSampler::chain_simple([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; let result = sampler.sample(&context, batch.n_tokens() - 1); @@ -900,13 +907,13 @@ fn raw_prompt_completion_with_timing(fixture: &LlamaFixture<'_>) -> Result<()> { } std::io::stderr().flush()?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens_list, 0, false)?; assert_eq!(classifier.pending_prompt_tokens(), prompt_token_count); assert_eq!(classifier.usage().prompt_tokens, 0); - ctx.decode(&mut batch) + ctx.decode(&batch) .with_context(|| "llama_decode() failed")?; let promoted = classifier.commit_prompt_tokens(); @@ -1034,13 +1041,13 @@ fn chat_inference_produces_coherent_output(fixture: &LlamaFixture<'_>) -> Result let tokens = model.str_to_token(&prompt, AddBos::Always, ParseSpecialTokens::Always)?; let prompt_token_count = u64::try_from(tokens.len())?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; assert_eq!(classifier.pending_prompt_tokens(), prompt_token_count); assert_eq!(classifier.usage().prompt_tokens, 0); - context.decode(&mut batch)?; + context.decode(&batch)?; let promoted = classifier.commit_prompt_tokens(); assert_eq!(promoted, prompt_token_count); @@ -1151,7 +1158,7 @@ fn json_schema_constrains_output(fixture: &LlamaFixture<'_>) -> Result<()> { let tokens_list = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); let last_index = i32::try_from(tokens_list.len())? - 1; for (index, token) in (0_i32..).zip(&tokens_list) { @@ -1163,7 +1170,7 @@ fn json_schema_constrains_output(fixture: &LlamaFixture<'_>) -> Result<()> { )?; } - ctx.decode(&mut batch)?; + ctx.decode(&batch)?; let schema = r#"{ "type": "object", @@ -1196,7 +1203,7 @@ fn json_schema_constrains_output(fixture: &LlamaFixture<'_>) -> Result<()> { batch.clear(); batch.add(&token, n_cur, &[0], true)?; n_cur += 1; - ctx.decode(&mut batch)?; + ctx.decode(&batch)?; } println!(); @@ -1585,9 +1592,9 @@ fn samples_token_constrained_by_grammar(fixture: &LlamaFixture<'_>) -> Result<() let prompt = "Answer yes or no:"; let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let llg_sampler = create_llg_sampler(model, "regex", REGEX_GRAMMAR)?; let mut chain = LlamaSampler::chain_simple([llg_sampler, LlamaSampler::greedy()?])?; @@ -1771,9 +1778,9 @@ fn llguidance_chain_samples_a_valid_token(fixture: &LlamaFixture<'_>) -> Result< )?; let tokens = model.str_to_token("Answer:", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let llg_sampler = create_llg_sampler(model, "regex", REGEX_GRAMMAR)?; let mut chain = LlamaSampler::chain_simple([llg_sampler, LlamaSampler::greedy()?])?; @@ -2071,7 +2078,12 @@ fn ingest_unmatched_prompt_tokens_does_not_record_usage(fixture: &LlamaFixture<' fn feed_prompt_to_batch_increments_pending_prompt_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { let model = fixture.model; let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(8)?, 1); + let mut batch = LlamaBatch::new(ContextLimits { + n_batch: 8, + n_ubatch: 8, + n_seq_max: 1, + n_vocab: model.n_vocab(), + }); classifier.feed_prompt_to_batch( &mut batch, @@ -2129,7 +2141,12 @@ fn feed_prompt_to_batch_increments_pending_prompt_tokens(fixture: &LlamaFixture< fn feed_prompt_sequence_to_batch_stages_all_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { let model = fixture.model; let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(8)?, 1); + let mut batch = LlamaBatch::new(ContextLimits { + n_batch: 8, + n_ubatch: 8, + n_seq_max: 1, + n_vocab: model.n_vocab(), + }); let tokens = vec![ model.token_bos().context("model declares a BOS token")?, @@ -2181,7 +2198,12 @@ fn commit_prompt_tokens_promotes_pending_count_to_usage_and_clears( ) -> Result<()> { let model = fixture.model; let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(8)?, 1); + let mut batch = LlamaBatch::new(ContextLimits { + n_batch: 8, + n_ubatch: 8, + n_seq_max: 1, + n_vocab: model.n_vocab(), + }); classifier.feed_prompt_to_batch( &mut batch, @@ -2244,7 +2266,12 @@ fn discard_pending_prompt_tokens_clears_count_without_recording_usage( ) -> Result<()> { let model = fixture.model; let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(8)?, 1); + let mut batch = LlamaBatch::new(ContextLimits { + n_batch: 8, + n_ubatch: 8, + n_seq_max: 1, + n_vocab: model.n_vocab(), + }); classifier.feed_prompt_to_batch( &mut batch, @@ -2346,10 +2373,10 @@ fn qwen35_grammar_with_a_token_reference_constrains_the_first_token( AddBos::Never, ParseSpecialTokens::Always, )?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(128)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&prompt_tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let mut sampler = LlamaSampler::chain_simple([ LlamaSampler::grammar(model, r#"root ::= "x""#, "root")?, diff --git a/llama-cpp-bindings-tests/tests/hidden_states.rs b/llama-cpp-bindings-tests/tests/hidden_states.rs index 2c66b1875..9461ded5f 100644 --- a/llama-cpp-bindings-tests/tests/hidden_states.rs +++ b/llama-cpp-bindings-tests/tests/hidden_states.rs @@ -2,12 +2,12 @@ use anyhow::Result; use llama_cpp_bindings::EmbeddingsError; use llama_cpp_bindings::SampledToken; use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::context::params::LlamaPoolingType; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::AddBos; use llama_cpp_bindings::model::ParseSpecialTokens; use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_bindings_types::BatchCapacity; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -40,7 +40,7 @@ fn hidden_state_context<'fixture>( } fn add_tokens( - batch: &mut LlamaBatch<'_>, + batch: &mut LlamaBatch, tokens: &[LlamaToken], first_position: i32, sequence_id: i32, @@ -66,12 +66,12 @@ fn hidden_state_after( branch: &[LlamaToken], ) -> Result> { let mut context = hidden_state_context(fixture)?; - let mut state_batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut state_batch = LlamaBatch::new(ContextLimits::of(&context)); add_tokens(&mut state_batch, state, 0, 0, false)?; - context.decode(&mut state_batch)?; + context.decode(&state_batch)?; - let mut branch_batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut branch_batch = LlamaBatch::new(ContextLimits::of(&context)); add_tokens( &mut branch_batch, @@ -80,7 +80,7 @@ fn hidden_state_after( 0, true, )?; - context.decode(&mut branch_batch)?; + context.decode(&branch_batch)?; Ok(context .nextn_embeddings_ith(branch_batch.n_tokens() - 1)? @@ -107,16 +107,16 @@ fn masked_nextn_embeddings_equal_the_final_hidden_states(fixture: &LlamaFixture< .with_embeddings(true) .with_pooling_type(LlamaPoolingType::None), )?; - let mut embedding_batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut embedding_batch = LlamaBatch::new(ContextLimits::of(&embedding_context)); embedding_batch.add_sequence(&tokens, 0, true)?; - embedding_context.decode(&mut embedding_batch)?; + embedding_context.decode(&embedding_batch)?; let mut nextn_context = hidden_state_context(fixture)?; - let mut nextn_batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut nextn_batch = LlamaBatch::new(ContextLimits::of(&nextn_context)); nextn_batch.add_sequence(&tokens, 0, true)?; - nextn_context.decode(&mut nextn_batch)?; + nextn_context.decode(&nextn_batch)?; assert_eq!( nextn_context.nextn_embeddings_ith(last_index)?, @@ -137,10 +137,10 @@ fn masked_nextn_embeddings_equal_the_final_hidden_states(fixture: &LlamaFixture< fn masked_nextn_embeddings_exist_only_for_output_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { let tokens = tokenize(fixture, STATE)?; let mut context = hidden_state_context(fixture)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); add_tokens(&mut batch, &tokens, 0, 0, true)?; - context.decode(&mut batch)?; + context.decode(&batch)?; assert_eq!( context.nextn_embeddings_ith(0), @@ -214,13 +214,13 @@ fn a_released_fork_leaves_its_shared_state_as_an_independent_decode_would( let branch_position = i32::try_from(state.len())?; let mut context = hidden_state_context(fixture)?; - let mut state_batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut state_batch = LlamaBatch::new(ContextLimits::of(&context)); add_tokens(&mut state_batch, &state, 0, 0, false)?; - context.decode(&mut state_batch)?; + context.decode(&state_batch)?; context.copy_kv_cache_seq(0, 1, None, None)?; - let mut first_branch_batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut first_branch_batch = LlamaBatch::new(ContextLimits::of(&context)); add_tokens( &mut first_branch_batch, @@ -229,7 +229,7 @@ fn a_released_fork_leaves_its_shared_state_as_an_independent_decode_would( 1, true, )?; - context.decode(&mut first_branch_batch)?; + context.decode(&first_branch_batch)?; let first_branch_hidden_state = context .nextn_embeddings_ith(first_branch_batch.n_tokens() - 1)? @@ -237,7 +237,7 @@ fn a_released_fork_leaves_its_shared_state_as_an_independent_decode_would( context.clear_kv_cache_seq(Some(1), None, None)?; - let mut second_branch_batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut second_branch_batch = LlamaBatch::new(ContextLimits::of(&context)); add_tokens( &mut second_branch_batch, @@ -246,7 +246,7 @@ fn a_released_fork_leaves_its_shared_state_as_an_independent_decode_would( 0, true, )?; - context.decode(&mut second_branch_batch)?; + context.decode(&second_branch_batch)?; assert_eq!( first_branch_hidden_state, @@ -292,10 +292,10 @@ fn decode_tokens( first_position: i32, output_on_last_token: bool, ) -> Result { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(context)); add_tokens(&mut batch, tokens, first_position, 0, output_on_last_token)?; - context.decode(&mut batch)?; + context.decode(&batch)?; Ok(batch.n_tokens() - 1) } diff --git a/llama-cpp-bindings-tests/tests/logits_availability.rs b/llama-cpp-bindings-tests/tests/logits_availability.rs new file mode 100644 index 000000000..76cc37c44 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/logits_availability.rs @@ -0,0 +1,80 @@ +use anyhow::Result; +use llama_cpp_bindings::LogitsError; +use llama_cpp_bindings::SampleError; +use llama_cpp_bindings::context::params::LlamaPoolingType; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +use crate::context_after_decoding_text::context_after_decoding_text; + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, + embeddings = true, +)] +fn a_pooled_embeddings_decode_produces_no_logits(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean)?; + + assert_eq!( + context.get_logits_ith(-1), + Err(LogitsError::PooledEmbeddingsProduceNoLogits { + pooling_type: LlamaPoolingType::Mean, + }) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, + embeddings = true, +)] +fn sampling_after_a_pooled_embeddings_decode_is_refused_instead_of_aborting( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean)?; + let mut sampler = LlamaSampler::greedy()?; + + assert_eq!( + sampler.sample(&context, -1), + Err(SampleError::Logits( + LogitsError::PooledEmbeddingsProduceNoLogits { + pooling_type: LlamaPoolingType::Mean, + } + )) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("LiquidAI/LFM2-ColBERT-350M-GGUF", "LFM2-ColBERT-350M-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, + embeddings = true, +)] +fn a_model_whose_embeddings_graph_has_no_logits_head_produces_no_logits( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let context = context_after_decoding_text(fixture, LlamaPoolingType::None)?; + + assert_eq!( + context.get_logits_ith(-1), + Err(LogitsError::GraphProducesNoLogits) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/main.rs b/llama-cpp-bindings-tests/tests/main.rs index ab13f7b94..52cb51445 100644 --- a/llama-cpp-bindings-tests/tests/main.rs +++ b/llama-cpp-bindings-tests/tests/main.rs @@ -3,6 +3,7 @@ reason = "harness trial functions must return Result to satisfy LlamaTestFn registration" )] mod backend_initialization; +mod batch_context_limits; mod chat_protocol; mod context_after_decoding_text; mod context_state; @@ -11,6 +12,7 @@ mod embedding_models; mod embedding_row_width; mod generation_control; mod hidden_states; +mod logits_availability; mod model_derived_data_caching; mod model_introspection; mod model_loading_errors; diff --git a/llama-cpp-bindings-tests/tests/model_introspection.rs b/llama-cpp-bindings-tests/tests/model_introspection.rs index 8340a3f6f..40f87f29d 100644 --- a/llama-cpp-bindings-tests/tests/model_introspection.rs +++ b/llama-cpp-bindings-tests/tests/model_introspection.rs @@ -384,13 +384,11 @@ fn rope_type_returns_none_for_qwen_default_models(fixture: &LlamaFixture<'_>) -> n_batch = 128, n_ubatch = 64, )] -fn vocab_type_returns_a_known_variant(fixture: &LlamaFixture<'_>) -> Result<()> { +fn default_models_use_a_byte_pair_encoding_vocabulary(fixture: &LlamaFixture<'_>) -> Result<()> { use llama_cpp_bindings::model::vocab_type::VocabType; - let vocab = fixture.model.vocab_type()?; - assert!( - matches!(vocab, VocabType::BPE | VocabType::SPM), - "vocab_type must be a known variant; got {vocab:?}" - ); + + assert_eq!(fixture.model.vocab_type()?, VocabType::Bpe); + Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/multimodal_audio.rs b/llama-cpp-bindings-tests/tests/multimodal_audio.rs index b0ba9392e..371c9744d 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_audio.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_audio.rs @@ -3,15 +3,16 @@ use anyhow::Result; use llama_cpp_bindings::BareJsonToolCalls; use llama_cpp_bindings::EvalMultimodalChunksParams; use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::LlamaChatMessage; use llama_cpp_bindings::mtmd::MtmdBitmap; +use llama_cpp_bindings::mtmd::MtmdCapability; use llama_cpp_bindings::mtmd::MtmdInputText; use llama_cpp_bindings::mtmd::mtmd_default_marker; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; -use llama_cpp_bindings_types::BatchCapacity; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -32,7 +33,7 @@ fn transcribe_audio(fixture: &LlamaFixture<'_>, audio_file_name: &str) -> Result .with_context(|| "unable to create llama context")?; assert!( - mtmd_ctx.support_audio(), + mtmd_ctx.capabilities(model).contains(MtmdCapability::Audio), "mmproj must support audio input for an audio transcription test" ); @@ -71,7 +72,7 @@ fn transcribe_audio(fixture: &LlamaFixture<'_>, audio_file_name: &str) -> Result .eval_multimodal_chunks( &chunks, mtmd_ctx, - &context, + &mut context, EvalMultimodalChunksParams { start_position: 0, seq_id: 0, @@ -99,7 +100,7 @@ fn transcribe_audio(fixture: &LlamaFixture<'_>, audio_file_name: &str) -> Result } let mut sampler = LlamaSampler::greedy()?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); let outcome = ClassifySampleLoop { classifier: &mut classifier, sampler: &mut sampler, @@ -170,7 +171,9 @@ fn audio_mmproj_reports_audio_support(fixture: &LlamaFixture<'_>) -> Result<()> .expect("mmproj_file declared in attribute"); assert!( - mtmd_ctx.support_audio(), + mtmd_ctx + .capabilities(fixture.model) + .contains(MtmdCapability::Audio), "an audio mmproj must report audio support" ); assert!( diff --git a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs index d82c8ac14..3172fd413 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs @@ -3,10 +3,12 @@ use anyhow::Result; use llama_cpp_bindings::BareJsonToolCalls; use llama_cpp_bindings::EvalMultimodalChunksParams; use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::LlamaChatMessage; use llama_cpp_bindings::model::LlamaModel; use llama_cpp_bindings::mtmd::MtmdBitmap; +use llama_cpp_bindings::mtmd::MtmdCapability; use llama_cpp_bindings::mtmd::MtmdContext; use llama_cpp_bindings::mtmd::MtmdInputText; use llama_cpp_bindings::mtmd::mtmd_default_marker; @@ -14,7 +16,6 @@ use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; -use llama_cpp_bindings_types::BatchCapacity; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -57,13 +58,12 @@ fn image_and_audio_together(fixture: &LlamaFixture<'_>) -> Result<()> { .mtmd_context .expect("mmproj_file declared in attribute"); - assert!( - mtmd_ctx.support_vision(), - "mmproj must support vision input for a combined image and audio test" - ); - assert!( - mtmd_ctx.support_audio(), - "mmproj must support audio input for a combined image and audio test" + assert_eq!( + mtmd_ctx.capabilities(model), + MtmdCapability::Chat + | MtmdCapability::Vision + | MtmdCapability::Audio + | MtmdCapability::TextGeneration ); let image_bitmap = load_fixture_bitmap(mtmd_ctx, "llamas.jpg")?; @@ -116,7 +116,7 @@ fn image_and_audio_together(fixture: &LlamaFixture<'_>) -> Result<()> { .eval_multimodal_chunks( &chunks, mtmd_ctx, - &context, + &mut context, EvalMultimodalChunksParams { start_position: 0, seq_id: 0, @@ -134,7 +134,7 @@ fn image_and_audio_together(fixture: &LlamaFixture<'_>) -> Result<()> { } let mut sampler = LlamaSampler::greedy()?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); let outcome = ClassifySampleLoop { classifier: &mut classifier, sampler: &mut sampler, diff --git a/llama-cpp-bindings-tests/tests/multimodal_vision.rs b/llama-cpp-bindings-tests/tests/multimodal_vision.rs index bc7662ecb..a4860f214 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_vision.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_vision.rs @@ -6,11 +6,14 @@ use llama_cpp_bindings::BareJsonToolCalls; use llama_cpp_bindings::EvalMultimodalChunksParams; use llama_cpp_bindings::TokenUsage; use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::error::EvalMultimodalChunksError; +use llama_cpp_bindings::error::sequence_id_out_of_range::SequenceIdOutOfRange; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::mtmd::FittedMtmdInputChunk; use llama_cpp_bindings::mtmd::FittedMtmdInputChunks; use llama_cpp_bindings::mtmd::MtmdBitmap; +use llama_cpp_bindings::mtmd::MtmdCapability; use llama_cpp_bindings::mtmd::MtmdContext; use llama_cpp_bindings::mtmd::MtmdContextParams; use llama_cpp_bindings::mtmd::MtmdEvalError; @@ -25,7 +28,6 @@ use llama_cpp_bindings_tests::build_user_prompt_with_media_marker::build_user_pr use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; -use llama_cpp_bindings_types::BatchCapacity; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -463,11 +465,16 @@ fn image_chunk_id_returns_some(fixture: &LlamaFixture<'_>) -> Result<()> { n_ubatch = 64, mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), )] -fn init_and_supports_vision(fixture: &LlamaFixture<'_>) -> Result<()> { +fn qwen_mmproj_reports_vision_only_chat_capabilities(fixture: &LlamaFixture<'_>) -> Result<()> { let mtmd_ctx = fixture .mtmd_context .expect("mmproj_file declared in attribute"); - assert!(mtmd_ctx.support_vision()); + + assert_eq!( + mtmd_ctx.capabilities(fixture.model), + MtmdCapability::Chat | MtmdCapability::Vision | MtmdCapability::TextGeneration + ); + Ok(()) } @@ -526,35 +533,6 @@ fn decode_use_mrope_is_true_for_qwen_vision(fixture: &LlamaFixture<'_>) -> Resul Ok(()) } -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn support_audio_is_false_for_vision_only_mmproj(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - assert!( - !mtmd_ctx.support_audio(), - "Qwen 3.5 / 3.6 mmproj is vision-only; support_audio must return false" - ); - Ok(()) -} - #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -608,13 +586,13 @@ fn eval_synthetic_bitmap(fixture: &LlamaFixture<'_>, width: u32, height: u32) -> ); } - let llama_ctx = LlamaContext::from_model( + let mut llama_ctx = LlamaContext::from_model( fixture.model, fixture.backend, (*fixture.context_params).into_llama_context_params(), )?; let n_batch = i32::try_from(llama_ctx.n_batch())?; - chunks.eval_chunks(mtmd_ctx, &llama_ctx, 0, 0, n_batch, false)?; + chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, false)?; Ok(()) } @@ -642,7 +620,7 @@ fn eval_chunks_returns_batch_size_exceeds_context_limit_for_huge_batch( let mtmd_ctx = fixture .mtmd_context .expect("mmproj_file declared in attribute"); - let llama_ctx = LlamaContext::from_model( + let mut llama_ctx = LlamaContext::from_model( fixture.model, fixture.backend, (*fixture.context_params).into_llama_context_params(), @@ -651,7 +629,7 @@ fn eval_chunks_returns_batch_size_exceeds_context_limit_for_huge_batch( let chunks = MtmdInputChunks::new()?; let huge_batch = i32::try_from(llama_ctx.n_batch() + 1)?; - let result = chunks.eval_chunks(mtmd_ctx, &llama_ctx, 0, 0, huge_batch, false); + let result = chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, huge_batch, false); assert_eq!( result, @@ -687,14 +665,14 @@ fn eval_chunks_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_> .mtmd_context .expect("mmproj_file declared in attribute"); let chunks = tokenize_synthetic(fixture, "Describe: <__media__>")?; - let llama_ctx = LlamaContext::from_model( + let mut llama_ctx = LlamaContext::from_model( fixture.model, fixture.backend, (*fixture.context_params).into_llama_context_params(), )?; assert_eq!( - chunks.eval_chunks(mtmd_ctx, &llama_ctx, 0, 0, 0, true), + chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, 0, true), Err(MtmdEvalError::NonPositiveBatchSize { requested: 0 }) ); assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); @@ -760,7 +738,7 @@ fn eval_single_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_> let mtmd_ctx = fixture .mtmd_context .expect("mmproj_file declared in attribute"); - let llama_ctx = LlamaContext::from_model( + let mut llama_ctx = LlamaContext::from_model( fixture.model, fixture.backend, (*fixture.context_params).into_llama_context_params(), @@ -774,7 +752,7 @@ fn eval_single_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_> FittedMtmdInputChunk::at(&fitted_chunks, 0).context("tokenization produced no chunks")?; assert_eq!( - first_chunk.eval_single(mtmd_ctx, &llama_ctx, 0, 0, 0, true), + first_chunk.eval_single(mtmd_ctx, &mut llama_ctx, 0, 0, 0, true), Err(MtmdEvalError::NonPositiveBatchSize { requested: 0 }) ); assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); @@ -782,6 +760,77 @@ fn eval_single_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_> Ok(()) } +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_single_rejects_a_batch_larger_than_the_context_batch_before_evaluating( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let mut llama_ctx = fixture.build_context()?; + let fitted_chunks = FittedMtmdInputChunks::fit( + tokenize_synthetic(fixture, "Describe: <__media__>")?, + mtmd_ctx, + llama_ctx.n_ubatch(), + )?; + let first_chunk = + FittedMtmdInputChunk::at(&fitted_chunks, 0).context("tokenization produced no chunks")?; + let context_max = llama_ctx.n_batch(); + let oversized_batch = i32::try_from(context_max + 1)?; + + assert_eq!( + first_chunk.eval_single(mtmd_ctx, &mut llama_ctx, 0, 0, oversized_batch, true), + Err(MtmdEvalError::BatchSizeExceedsContextLimit { + requested: oversized_batch, + context_max, + }) + ); + assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_chunks_rejects_a_sequence_the_context_does_not_have_instead_of_aborting( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let mut llama_ctx = fixture.build_context()?; + let chunks = tokenize_synthetic(fixture, "Describe: <__media__>")?; + let n_seq_max = llama_ctx.n_seq_max(); + let missing_sequence = i32::try_from(n_seq_max)?; + let n_batch = i32::try_from(llama_ctx.n_batch())?; + + assert_eq!( + chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, missing_sequence, n_batch, true), + Err(MtmdEvalError::SequenceIdOutOfRange(SequenceIdOutOfRange { + seq_id: missing_sequence, + n_seq_max, + })) + ); + assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); + + Ok(()) +} + #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -806,7 +855,7 @@ fn eval_single_rejects_a_micro_batch_smaller_than_the_fitted_one_before_evaluati let mtmd_ctx = fixture .mtmd_context .expect("mmproj_file declared in attribute"); - let llama_ctx = LlamaContext::from_model( + let mut llama_ctx = LlamaContext::from_model( fixture.model, fixture.backend, (*fixture.context_params).into_llama_context_params(), @@ -824,7 +873,7 @@ fn eval_single_rejects_a_micro_batch_smaller_than_the_fitted_one_before_evaluati assert_eq!( first_chunk.eval_single( mtmd_ctx, - &llama_ctx, + &mut llama_ctx, 0, 0, i32::try_from(smaller_micro_batch_tokens)?, @@ -863,7 +912,7 @@ fn classifier_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_>) .mtmd_context .expect("mmproj_file declared in attribute"); let chunks = tokenize_synthetic(fixture, "Describe: <__media__>")?; - let llama_ctx = LlamaContext::from_model( + let mut llama_ctx = LlamaContext::from_model( fixture.model, fixture.backend, (*fixture.context_params).into_llama_context_params(), @@ -876,7 +925,7 @@ fn classifier_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_>) classifier.eval_multimodal_chunks( &chunks, mtmd_ctx, - &llama_ctx, + &mut llama_ctx, EvalMultimodalChunksParams { start_position: 0, seq_id: 0, @@ -938,13 +987,13 @@ fn eval_chunks_with_standard_image(fixture: &LlamaFixture<'_>) -> Result<()> { required_n_ctx, ); - let llama_ctx = LlamaContext::from_model( + let mut llama_ctx = LlamaContext::from_model( fixture.model, fixture.backend, (*fixture.context_params).into_llama_context_params(), )?; let n_batch = i32::try_from(llama_ctx.n_batch())?; - let result = chunks.eval_chunks(mtmd_ctx, &llama_ctx, 0, 0, n_batch, false); + let result = chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, false); assert!(result.is_ok()); @@ -1144,7 +1193,9 @@ fn multimodal_vision_inference_produces_output(fixture: &LlamaFixture<'_>) -> Re .with_context(|| "unable to create llama context")?; assert!( - mtmd_ctx.support_vision(), + mtmd_ctx + .capabilities(model) + .contains(MtmdCapability::Vision), "model should support vision input" ); @@ -1193,7 +1244,7 @@ fn multimodal_vision_inference_produces_output(fixture: &LlamaFixture<'_>) -> Re .eval_multimodal_chunks( &chunks, mtmd_ctx, - &ctx, + &mut ctx, EvalMultimodalChunksParams { start_position: 0, seq_id: 0, @@ -1213,7 +1264,7 @@ fn multimodal_vision_inference_produces_output(fixture: &LlamaFixture<'_>) -> Re } let mut sampler = LlamaSampler::greedy()?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); let totals = ClassifySampleLoop { classifier: &mut classifier, sampler: &mut sampler, @@ -1274,13 +1325,13 @@ fn build_multimodal_chunks_and_eval_into_usage( let expected = ChunkTokenBreakdown::from_chunks(&chunks)?; let context_params = (*fixture.context_params).into_llama_context_params(); - let context = LlamaContext::from_model(model, fixture.backend, context_params)?; + let mut context = LlamaContext::from_model(model, fixture.backend, context_params)?; let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; classifier.eval_multimodal_chunks( &chunks, mtmd_ctx, - &context, + &mut context, EvalMultimodalChunksParams { start_position: 0, seq_id: 0, @@ -1629,7 +1680,7 @@ fn gemma4_classifier_emits_reasoning_for_multimodal_thinking_prompt( let n_past = classifier.eval_multimodal_chunks( &chunks, mtmd_ctx, - &context, + &mut context, EvalMultimodalChunksParams { start_position: 0, seq_id: 0, @@ -1647,7 +1698,7 @@ fn gemma4_classifier_emits_reasoning_for_multimodal_thinking_prompt( LlamaSampler::dist(0x00C0_FFEE)?, ])?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); let outcome = ClassifySampleLoop { classifier: &mut classifier, sampler: &mut sampler, @@ -1732,7 +1783,7 @@ fn mistral3_classifier_emits_reasoning_for_multimodal_thinking_prompt( let n_past = classifier.eval_multimodal_chunks( &chunks, mtmd_ctx, - &context, + &mut context, EvalMultimodalChunksParams { start_position: 0, seq_id: 0, @@ -1742,7 +1793,7 @@ fn mistral3_classifier_emits_reasoning_for_multimodal_thinking_prompt( )?; let mut sampler = LlamaSampler::greedy()?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); let outcome = ClassifySampleLoop { classifier: &mut classifier, sampler: &mut sampler, @@ -1828,7 +1879,7 @@ fn qwen35_classifier_emits_reasoning_for_multimodal_thinking_prompt( let n_past = classifier.eval_multimodal_chunks( &chunks, mtmd_ctx, - &context, + &mut context, EvalMultimodalChunksParams { start_position: 0, seq_id: 0, @@ -1846,7 +1897,7 @@ fn qwen35_classifier_emits_reasoning_for_multimodal_thinking_prompt( LlamaSampler::dist(0x00C0_FFEE)?, ])?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); let outcome = ClassifySampleLoop { classifier: &mut classifier, sampler: &mut sampler, @@ -1923,7 +1974,7 @@ fn qwen36_classifier_emits_reasoning_for_multimodal_thinking_prompt( let n_past = classifier.eval_multimodal_chunks( &chunks, mtmd_ctx, - &context, + &mut context, EvalMultimodalChunksParams { start_position: 0, seq_id: 0, @@ -1941,7 +1992,7 @@ fn qwen36_classifier_emits_reasoning_for_multimodal_thinking_prompt( LlamaSampler::dist(0x00C0_FFEE)?, ])?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); let outcome = ClassifySampleLoop { classifier: &mut classifier, sampler: &mut sampler, @@ -2005,25 +2056,21 @@ fn gemma3_eval_chunks_rejects_a_non_causal_image_larger_than_the_micro_batch_bef .expect("mmproj_file declared in attribute"); let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; let image_tokens = ChunkTokenBreakdown::from_chunks(&chunks)?.image; - let llama_ctx = LlamaContext::from_model( + let mut llama_ctx = LlamaContext::from_model( fixture.model, fixture.backend, (*fixture.context_params).into_llama_context_params(), )?; + let n_batch = i32::try_from(llama_ctx.n_batch())?; + let micro_batch_tokens = llama_ctx.n_ubatch(); + assert_eq!( - chunks.eval_chunks( - mtmd_ctx, - &llama_ctx, - 0, - 0, - i32::try_from(llama_ctx.n_batch())?, - true - ), + chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, true), Err(MtmdEvalError::NonCausalChunkExceedsMicroBatch( NonCausalChunkMicroBatchMismatch { chunk_tokens: usize::try_from(image_tokens)?, - micro_batch_tokens: llama_ctx.n_ubatch(), + micro_batch_tokens, } )) ); @@ -2087,7 +2134,7 @@ fn gemma3_classifier_leaves_the_prompt_untouched_when_a_non_causal_image_exceeds .mtmd_context .expect("mmproj_file declared in attribute"); let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; - let llama_ctx = LlamaContext::from_model( + let mut llama_ctx = LlamaContext::from_model( fixture.model, fixture.backend, (*fixture.context_params).into_llama_context_params(), @@ -2096,14 +2143,15 @@ fn gemma3_classifier_leaves_the_prompt_untouched_when_a_non_causal_image_exceeds .model .sampled_token_classifier(BareJsonToolCalls::Ignore)?; + let n_batch = i32::try_from(llama_ctx.n_batch())?; let evaluation = classifier.eval_multimodal_chunks( &chunks, mtmd_ctx, - &llama_ctx, + &mut llama_ctx, EvalMultimodalChunksParams { start_position: 0, seq_id: 0, - n_batch: i32::try_from(llama_ctx.n_batch())?, + n_batch, logits_last: true, }, ); diff --git a/llama-cpp-bindings-tests/tests/structured_chat_output.rs b/llama-cpp-bindings-tests/tests/structured_chat_output.rs index 6796f5b60..1de7a11cc 100644 --- a/llama-cpp-bindings-tests/tests/structured_chat_output.rs +++ b/llama-cpp-bindings-tests/tests/structured_chat_output.rs @@ -14,6 +14,7 @@ use llama_cpp_bindings::TokenUsage; use llama_cpp_bindings::ToolCallArgsShape; use llama_cpp_bindings::ToolCallArguments; use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::AddBos; use llama_cpp_bindings::model::LlamaChatMessage; @@ -22,7 +23,6 @@ use llama_cpp_bindings::model::ParseSpecialTokens; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoopOutcome; -use llama_cpp_bindings_types::BatchCapacity; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; use serde_json::Value; @@ -80,16 +80,16 @@ fn deepseek_r1_8b_classifier_does_not_emit_reasoning_for_thinking_disabled_promp )?; let prompt_token_count = u64::try_from(prompt_tokens.len())?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - let mut context = LlamaContext::from_model( model, backend, (*fixture.context_params).into_llama_context_params(), )?; - context.decode(&mut batch)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; + + context.decode(&batch)?; let promoted = classifier.commit_prompt_tokens(); assert_eq!(promoted, prompt_token_count); @@ -193,16 +193,16 @@ fn deepseek_r1_8b_classifier_emits_reasoning_for_thinking_enabled_prompt( )?; let prompt_token_count = u64::try_from(prompt_tokens.len())?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - let mut context = LlamaContext::from_model( model, backend, (*fixture.context_params).into_llama_context_params(), )?; - context.decode(&mut batch)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; + + context.decode(&batch)?; let promoted = classifier.commit_prompt_tokens(); assert_eq!(promoted, prompt_token_count); @@ -650,16 +650,16 @@ fn gemma4_classifier_does_not_emit_reasoning_for_thinking_disabled_prompt( )?; let prompt_token_count = u64::try_from(prompt_tokens.len())?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - let mut context = LlamaContext::from_model( model, backend, (*fixture.context_params).into_llama_context_params(), )?; - context.decode(&mut batch)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; + + context.decode(&batch)?; let promoted = classifier.commit_prompt_tokens(); assert_eq!(promoted, prompt_token_count); @@ -754,16 +754,16 @@ fn gemma4_classifier_emits_reasoning_for_thinking_prompt(fixture: &LlamaFixture< )?; let prompt_token_count = u64::try_from(prompt_tokens.len())?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - let mut context = LlamaContext::from_model( model, backend, (*fixture.context_params).into_llama_context_params(), )?; - context.decode(&mut batch)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; + + context.decode(&batch)?; let promoted = classifier.commit_prompt_tokens(); assert_eq!(promoted, prompt_token_count); @@ -963,16 +963,16 @@ What is 2 + 2? )?; let prompt_token_count = u64::try_from(prompt_tokens.len())?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - let mut context = LlamaContext::from_model( model, backend, (*fixture.context_params).into_llama_context_params(), )?; - context.decode(&mut batch)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; + + context.decode(&batch)?; let promoted = classifier.commit_prompt_tokens(); assert_eq!(promoted, prompt_token_count); @@ -1046,16 +1046,16 @@ What is 2 + 2? )?; let prompt_token_count = u64::try_from(prompt_tokens.len())?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - let mut context = LlamaContext::from_model( model, backend, (*fixture.context_params).into_llama_context_params(), )?; - context.decode(&mut batch)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; + + context.decode(&batch)?; let promoted = classifier.commit_prompt_tokens(); assert_eq!(promoted, prompt_token_count); @@ -1227,16 +1227,16 @@ fn mistral3_classifier_does_not_emit_reasoning_for_thinking_disabled_prompt( )?; let prompt_token_count = u64::try_from(prompt_tokens.len())?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - let mut context = LlamaContext::from_model( model, backend, (*fixture.context_params).into_llama_context_params(), )?; - context.decode(&mut batch)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; + + context.decode(&batch)?; let promoted = classifier.commit_prompt_tokens(); assert_eq!(promoted, prompt_token_count); @@ -1307,16 +1307,16 @@ to the user.[/THINK]Here, provide a self-contained response.[/SYSTEM_PROMPT]\ )?; let prompt_token_count = u64::try_from(prompt_tokens.len())?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - let mut context = LlamaContext::from_model( model, backend, (*fixture.context_params).into_llama_context_params(), )?; - context.decode(&mut batch)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; + + context.decode(&batch)?; let promoted = classifier.commit_prompt_tokens(); assert_eq!(promoted, prompt_token_count); @@ -1441,10 +1441,10 @@ fn qwen35_chat_inference_emits_reasoning_when_template_auto_opens( let tokens = model.str_to_token(&prompt, AddBos::Always, ParseSpecialTokens::Always)?; let prompt_token_count = u64::try_from(tokens.len())?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let promoted = classifier.commit_prompt_tokens(); assert_eq!(promoted, prompt_token_count); @@ -1595,16 +1595,16 @@ What is 2 + 2?<|im_end|> )?; let prompt_token_count = u64::try_from(prompt_tokens.len())?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - let mut context = LlamaContext::from_model( model, backend, (*fixture.context_params).into_llama_context_params(), )?; - context.decode(&mut batch)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; + + context.decode(&batch)?; let promoted = classifier.commit_prompt_tokens(); assert_eq!(promoted, prompt_token_count); @@ -1678,16 +1678,16 @@ What is 2 + 2?<|im_end|> )?; let prompt_token_count = u64::try_from(prompt_tokens.len())?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - let mut context = LlamaContext::from_model( model, backend, (*fixture.context_params).into_llama_context_params(), )?; - context.decode(&mut batch)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; + + context.decode(&batch)?; let promoted = classifier.commit_prompt_tokens(); assert_eq!(promoted, prompt_token_count); @@ -2072,10 +2072,10 @@ fn qwen36_chat_inference_emits_reasoning_when_template_auto_opens( let tokens = model.str_to_token(&prompt, AddBos::Always, ParseSpecialTokens::Always)?; let prompt_token_count = u64::try_from(tokens.len())?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; let promoted = classifier.commit_prompt_tokens(); assert_eq!(promoted, prompt_token_count); @@ -2145,16 +2145,16 @@ What is 2 + 2?<|im_end|> )?; let prompt_token_count = u64::try_from(prompt_tokens.len())?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - let mut context = LlamaContext::from_model( model, backend, (*fixture.context_params).into_llama_context_params(), )?; - context.decode(&mut batch)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; + + context.decode(&batch)?; let promoted = classifier.commit_prompt_tokens(); assert_eq!(promoted, prompt_token_count); @@ -2228,16 +2228,16 @@ What is 2 + 2?<|im_end|> )?; let prompt_token_count = u64::try_from(prompt_tokens.len())?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - let mut context = LlamaContext::from_model( model, backend, (*fixture.context_params).into_llama_context_params(), )?; - context.decode(&mut batch)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; + + context.decode(&batch)?; let promoted = classifier.commit_prompt_tokens(); assert_eq!(promoted, prompt_token_count); diff --git a/llama-cpp-bindings-types/src/batch_capacity.rs b/llama-cpp-bindings-types/src/batch_capacity.rs deleted file mode 100644 index 7c4efbf81..000000000 --- a/llama-cpp-bindings-types/src/batch_capacity.rs +++ /dev/null @@ -1,109 +0,0 @@ -use std::num::NonZeroU16; -use std::num::NonZeroU32; - -use crate::batch_capacity_error::BatchCapacityError; - -const MAX_TOKENS: u32 = i32::MAX.unsigned_abs(); - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct BatchCapacity(NonZeroU32); - -impl BatchCapacity { - #[must_use] - pub fn at_most(self, tokens: NonZeroU32) -> Self { - Self(self.0.min(tokens)) - } - - #[must_use] - pub const fn tokens(self) -> NonZeroU32 { - self.0 - } - - #[must_use] - pub const fn tokens_i32(self) -> i32 { - self.0.get().cast_signed() - } - - #[must_use] - pub const fn tokens_usize(self) -> usize { - self.0.get() as usize - } -} - -impl From for BatchCapacity { - fn from(tokens: NonZeroU16) -> Self { - Self(NonZeroU32::from(tokens)) - } -} - -impl TryFrom for BatchCapacity { - type Error = BatchCapacityError; - - fn try_from(tokens: u32) -> Result { - if tokens > MAX_TOKENS { - return Err(BatchCapacityError::TooLarge { - tokens, - max_tokens: MAX_TOKENS, - }); - } - - NonZeroU32::new(tokens) - .map(Self) - .ok_or(BatchCapacityError::Zero) - } -} - -#[cfg(test)] -mod tests { - use std::num::NonZeroU16; - use std::num::NonZeroU32; - - use super::BatchCapacity; - use crate::batch_capacity_error::BatchCapacityError; - - #[test] - fn rejects_a_batch_without_tokens() { - assert_eq!(BatchCapacity::try_from(0), Err(BatchCapacityError::Zero)); - } - - #[test] - fn rejects_a_batch_llama_cpp_cannot_address() { - assert_eq!( - BatchCapacity::try_from(u32::MAX), - Err(BatchCapacityError::TooLarge { - tokens: u32::MAX, - max_tokens: i32::MAX.unsigned_abs(), - }) - ); - } - - #[test] - fn the_largest_batch_keeps_its_size_in_every_width() { - let capacity = BatchCapacity::try_from(i32::MAX.unsigned_abs()).unwrap(); - - assert_eq!(capacity.tokens().get(), i32::MAX.unsigned_abs()); - assert_eq!(capacity.tokens_i32(), i32::MAX); - assert_eq!(capacity.tokens_usize(), i32::MAX.unsigned_abs() as usize); - } - - #[test] - fn a_batch_never_exceeds_the_token_bound_it_is_capped_at() { - let capacity = BatchCapacity::try_from(2048).unwrap(); - - assert_eq!( - [ - capacity.at_most(NonZeroU32::new(512).unwrap()), - capacity.at_most(NonZeroU32::MAX), - ], - [BatchCapacity::try_from(512).unwrap(), capacity] - ); - } - - #[test] - fn every_nonzero_sixteen_bit_size_is_a_valid_batch() { - assert_eq!( - BatchCapacity::from(NonZeroU16::MAX), - BatchCapacity::try_from(u32::from(u16::MAX)).unwrap() - ); - } -} diff --git a/llama-cpp-bindings-types/src/batch_capacity_error.rs b/llama-cpp-bindings-types/src/batch_capacity_error.rs deleted file mode 100644 index 4eb5f857d..000000000 --- a/llama-cpp-bindings-types/src/batch_capacity_error.rs +++ /dev/null @@ -1,7 +0,0 @@ -#[derive(Debug, Eq, PartialEq, thiserror::Error)] -pub enum BatchCapacityError { - #[error("a batch must hold at least one token")] - Zero, - #[error("a batch of {tokens} tokens exceeds the {max_tokens} tokens llama.cpp can address")] - TooLarge { tokens: u32, max_tokens: u32 }, -} diff --git a/llama-cpp-bindings-types/src/lib.rs b/llama-cpp-bindings-types/src/lib.rs index a35133e72..09f82ca00 100644 --- a/llama-cpp-bindings-types/src/lib.rs +++ b/llama-cpp-bindings-types/src/lib.rs @@ -3,8 +3,6 @@ deny(clippy::unwrap_used, clippy::expect_used, clippy::panic) )] -pub mod batch_capacity; -pub mod batch_capacity_error; pub mod bracketed_json_shape; pub mod json_object_shape; pub mod key_value_xml_tags_shape; @@ -23,8 +21,6 @@ pub mod tool_call_markers; pub mod tool_call_value_quote; pub mod xml_tags_shape; -pub use batch_capacity::BatchCapacity; -pub use batch_capacity_error::BatchCapacityError; pub use bracketed_json_shape::BracketedJsonShape; pub use json_object_shape::JsonObjectShape; pub use key_value_xml_tags_shape::KeyValueXmlTagsShape; diff --git a/llama-cpp-bindings/src/batch_add_error.rs b/llama-cpp-bindings/src/batch_add_error.rs index 146743362..d705ca403 100644 --- a/llama-cpp-bindings/src/batch_add_error.rs +++ b/llama-cpp-bindings/src/batch_add_error.rs @@ -1,22 +1,16 @@ -use std::num::TryFromIntError; +use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; +use crate::error::token_outside_vocabulary::TokenOutsideVocabulary; -#[derive(thiserror::Error, Debug, PartialEq, Eq)] +#[derive(thiserror::Error, Clone, Debug, PartialEq, Eq)] pub enum BatchAddError { - #[error("Insufficient Space of {0}")] - InsufficientSpace(usize), - #[error("Empty buffer")] - EmptyBuffer, - #[error( - "a token in {sequence_ids} sequences does not fit a batch created for at most {n_seq_max} sequences per token" - )] - TooManySequenceIds { sequence_ids: i32, n_seq_max: i32 }, + #[error("the batch already holds the {n_batch} tokens one decode of the context takes")] + BatchFull { n_batch: u32 }, + #[error(transparent)] + TokenOutsideVocabulary(#[from] TokenOutsideVocabulary), + #[error("a token must belong to at least one sequence")] + NoSequenceIds, + #[error(transparent)] + SequenceIdOutOfRange(#[from] SequenceIdOutOfRange), #[error("the batch already holds {n_tokens} tokens and one more would overflow i32")] TokenCountOverflow { n_tokens: i32 }, - #[error("{value_description} does not fit into {target_type}")] - IntegerOverflow { - value_description: &'static str, - target_type: &'static str, - #[source] - source: TryFromIntError, - }, } diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index ac4c4a223..1ae604972 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -11,9 +11,13 @@ use std::sync::atomic::Ordering; use llama_cpp_ffi_status::read_and_free_cpp_string; +use crate::context::context_limits::ContextLimits; +use crate::context::decoded_output_rows::DecodedOutputRows; +use crate::context::decoded_outputs::DecodedOutputs; use crate::context::params::LlamaContextParams; use crate::context::params::LlamaPoolingType; use crate::context::sequence_output_capacity::SequenceOutputCapacity; +use crate::error::batch_transfer_error::BatchTransferError; use crate::llama_backend::LlamaBackend; use crate::llama_batch::LlamaBatch; use crate::model::LlamaModel; @@ -154,6 +158,137 @@ fn new_context_with_model_status_to_result( } } +fn batch_ext_init_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_batch_ext_init_status, + out_batch: *mut llama_cpp_bindings_sys::llama_batch_ext, + out_error: *mut c_char, +) -> Result, LlamaContextLoadError> { + match status { + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_OK => { + NonNull::new(out_batch).ok_or_else(|| { + crate::FfiContractError { + operation: "llama_rs_batch_ext_init", + detail: "success status contained a null batch", + } + .into() + }) + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_ERROR_STRING_ALLOCATION_FAILED => { + Err(LlamaContextLoadError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_LLAMA_CPP_OUT_OF_MEMORY => { + Err(LlamaContextLoadError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + "llama_rs_batch_ext_init", + "reported a thrown C++ exception without an error message", + ) + }?; + Err(LlamaContextLoadError::Reported { message }) + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_NULL_CTX_ARG => { + Err(crate::FfiContractError { + operation: "llama_rs_batch_ext_init", + detail: "was given a null ctx argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_NULL_OUT_BATCH_ARG => { + Err(crate::FfiContractError { + operation: "llama_rs_batch_ext_init", + detail: "was given a null out_batch argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_NULL_OUT_ERROR_ARG => { + Err(crate::FfiContractError { + operation: "llama_rs_batch_ext_init", + detail: "was given a null out_error argument", + } + .into()) + } + other => Err(crate::FfiStatusError { + operation: "llama_rs_batch_ext_init", + code: i64::from(other), + } + .into()), + } +} + +fn batch_ext_add_entry_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_batch_ext_add_entry_status, + out_error: *mut c_char, +) -> Result<(), BatchTransferError> { + let refused_validated_entry = |detail: &'static str| -> Result<(), BatchTransferError> { + Err(crate::FfiContractError { + operation: "llama_rs_batch_ext_add_entry", + detail, + } + .into()) + }; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_OK => Ok(()), + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_ERROR_STRING_ALLOCATION_FAILED => { + Err(BatchTransferError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_OUT_OF_MEMORY => { + Err(BatchTransferError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + "llama_rs_batch_ext_add_entry", + "reported a thrown C++ exception without an error message", + ) + }?; + Err(BatchTransferError::Reported { message }) + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_BATCH_FULL => refused_validated_entry( + "refused an entry of a batch validated against its limits as full", + ), + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_INVALID_TOKEN => { + refused_validated_entry("refused a token validated against the vocabulary") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_INVALID_SEQUENCE_ID => { + refused_validated_entry("refused a sequence id validated against the context") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_RETURNED_UNKNOWN_CODE => { + refused_validated_entry("refused an entry with an undocumented code") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_POSITION_REJECTED => { + refused_validated_entry("refused the position of an entry it had just added") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_OUTPUT_REJECTED => { + refused_validated_entry("refused the output flag of an entry it had just added") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_EMPTY_SEQ_IDS => { + refused_validated_entry("was given an entry without sequence ids") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_BATCH_ARG => { + refused_validated_entry("was given a null batch argument") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_SEQ_IDS_ARG => { + refused_validated_entry("was given a null seq_ids argument") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG => { + refused_validated_entry("was given a null out_llama_cpp_return_code argument") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_OUT_ERROR_ARG => { + refused_validated_entry("was given a null out_error argument") + } + other => Err(crate::FfiStatusError { + operation: "llama_rs_batch_ext_add_entry", + code: i64::from(other), + } + .into()), + } +} + fn decode_status_to_result( status: llama_cpp_bindings_sys::llama_rs_decode_status, out_llama_cpp_return_code: i32, @@ -194,6 +329,11 @@ fn decode_status_to_result( detail: "was given a null ctx argument", } .into()), + llama_cpp_bindings_sys::LLAMA_RS_DECODE_NULL_BATCH_ARG => Err(crate::FfiContractError { + operation: "llama_rs_decode", + detail: "was given a null batch argument", + } + .into()), llama_cpp_bindings_sys::LLAMA_RS_DECODE_NULL_OUT_ERROR_ARG => { Err(crate::FfiContractError { operation: "llama_rs_decode", @@ -252,6 +392,11 @@ fn encode_status_to_result( detail: "was given a null ctx argument", } .into()), + llama_cpp_bindings_sys::LLAMA_RS_ENCODE_NULL_BATCH_ARG => Err(crate::FfiContractError { + operation: "llama_rs_encode", + detail: "was given a null batch argument", + } + .into()), llama_cpp_bindings_sys::LLAMA_RS_ENCODE_NULL_MODEL => Err(crate::FfiContractError { operation: "llama_rs_encode", detail: "was given a null model argument", @@ -265,35 +410,18 @@ fn encode_status_to_result( } } -fn token_index_within_context(token_index: i32, context_size: u32) -> Result<(), LogitsError> { - if token_index >= 0 { - let token_index_u32 = - u32::try_from(token_index).map_err(LogitsError::TokenIndexOverflow)?; - - if context_size <= token_index_u32 { - return Err(LogitsError::TokenIndexExceedsContext { - token_index: token_index_u32, - context_size, - }); - } - } - - Ok(()) -} - -unsafe fn logits_slice_from_raw_parts<'logits>( - data: *const f32, +unsafe fn logits_slice<'logits>( + logits: NonNull, n_vocab: i32, ) -> Result<&'logits [f32], LogitsError> { - if data.is_null() { - return Err(LogitsError::NullLogits); - } - let len = usize::try_from(n_vocab).map_err(LogitsError::VocabSizeOverflow)?; - Ok(unsafe { slice::from_raw_parts(data, len) }) + Ok(unsafe { slice::from_raw_parts(logits.as_ptr(), len) }) } +pub mod context_limits; +pub mod decoded_output_rows; +pub mod decoded_outputs; pub mod kv_cache; pub mod kv_cache_type; pub mod llama_attention_type; @@ -320,7 +448,8 @@ pub struct LlamaContext<'model> { pub context: NonNull, pub model: &'model LlamaModel, abort_flag: Option>, - initialized_logits: Vec, + decoded_outputs: DecodedOutputs, + batch_ext: NonNull, } impl Debug for LlamaContext<'_> { @@ -332,17 +461,38 @@ impl Debug for LlamaContext<'_> { } impl<'model> LlamaContext<'model> { - #[must_use] - pub const fn new( - llama_model: &'model LlamaModel, - llama_context: NonNull, - ) -> Self { - Self { - context: llama_context, - model: llama_model, + fn with_batch( + model: &'model LlamaModel, + context: NonNull, + ) -> Result { + let mut out_batch: *mut llama_cpp_bindings_sys::llama_batch_ext = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_batch_ext_init( + context.as_ptr(), + &raw mut out_batch, + &raw mut out_error, + ) + }; + let batch_ext = batch_ext_init_status_to_result(status, out_batch, out_error).inspect_err( + |_batch_init_failure| unsafe { + llama_cpp_bindings_sys::llama_free(context.as_ptr()); + }, + )?; + let pooling_type = LlamaPoolingType::from(unsafe { + llama_cpp_bindings_sys::llama_pooling_type(context.as_ptr()) + }); + + Ok(Self { + context, + model, abort_flag: None, - initialized_logits: Vec::new(), - } + decoded_outputs: DecodedOutputs { + rows: DecodedOutputRows::BatchIndices(Vec::new()), + pooling_type, + }, + batch_ext, + }) } /// # Errors @@ -387,7 +537,7 @@ impl<'model> LlamaContext<'model> { }; let context = new_context_with_model_status_to_result(status, out_ctx, out_error)?; - Ok(Self::new(model, context)) + Self::with_batch(model, context) } #[must_use] @@ -400,6 +550,20 @@ impl<'model> LlamaContext<'model> { unsafe { llama_cpp_bindings_sys::llama_n_ubatch(self.context.as_ptr()) } } + #[must_use] + pub fn n_seq_max(&self) -> u32 { + unsafe { llama_cpp_bindings_sys::llama_n_seq_max(self.context.as_ptr()) } + } + + fn decodes_batches_in_one_micro_batch(&self) -> bool { + let keeps_no_memory = + unsafe { llama_cpp_bindings_sys::llama_get_memory(self.context.as_ptr()) }.is_null(); + let attends_causally = + unsafe { llama_cpp_bindings_sys::llama_get_causal_attn(self.context.as_ptr()) }; + + keeps_no_memory || !attends_causally + } + fn exceeds_micro_batch(&self, batch: &LlamaBatch) -> bool { i64::from(batch.n_tokens()) > i64::from(self.n_ubatch()) } @@ -451,41 +615,77 @@ impl<'model> LlamaContext<'model> { unsafe { llama_cpp_bindings_sys::llama_detach_threadpool(self.context.as_ptr()) } } - pub fn mark_logits_initialized(&mut self, token_index: i32) { - self.initialized_logits = vec![token_index]; + /// Records which rows the last decode wrote, for a decode llama.cpp ran on this context + /// without one of this context's batches, such as the mtmd helper decoding media chunks. + pub fn record_decoded_rows(&mut self, rows: DecodedOutputRows) { + self.decoded_outputs = DecodedOutputs { + rows, + pooling_type: self.pooling_type(), + }; + } + + fn transfer_batch(&self, batch: &LlamaBatch) -> Result<(), BatchTransferError> { + let context_limits = ContextLimits::of(self); + + if batch.limits() != context_limits { + return Err(BatchTransferError::LimitsMismatch { + batch: batch.limits(), + context: context_limits, + }); + } + + unsafe { llama_cpp_bindings_sys::llama_batch_ext_clear(self.batch_ext.as_ptr()) }; + + for entry in batch.entries() { + let mut out_llama_cpp_return_code: i32 = 0; + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_batch_ext_add_entry( + self.batch_ext.as_ptr(), + entry.token.0, + entry.pos, + entry.seq_ids.as_ptr(), + entry.seq_ids.len(), + entry.output, + &raw mut out_llama_cpp_return_code, + &raw mut out_error, + ) + }; + + batch_ext_add_entry_status_to_result(status, out_error)?; + } + + Ok(()) } /// # Errors /// /// - `DecodeError` if the decoding failed. - pub fn decode(&mut self, batch: &mut LlamaBatch) -> Result<(), DecodeError> { - let decodes_batches_in_one_micro_batch = unsafe { - llama_cpp_bindings_sys::llama_rs_context_decodes_batches_in_one_micro_batch( - self.context.as_ptr(), - ) - }; - - if decodes_batches_in_one_micro_batch && self.exceeds_micro_batch(batch) { + pub fn decode(&mut self, batch: &LlamaBatch) -> Result<(), DecodeError> { + if self.decodes_batches_in_one_micro_batch() && self.exceeds_micro_batch(batch) { return Err(DecodeError::BatchExceedsMicroBatch { n_tokens: batch.n_tokens(), n_ubatch: self.n_ubatch(), }); } + self.transfer_batch(batch)?; + let mut out_llama_cpp_return_code: i32 = 0; let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_decode( self.context.as_ptr(), - batch.llama_batch, + self.batch_ext.as_ptr(), &raw mut out_llama_cpp_return_code, &raw mut out_error, ) }; decode_status_to_result(status, out_llama_cpp_return_code, out_error)?; - self.initialized_logits - .clone_from(&batch.initialized_logits); + self.record_decoded_rows(DecodedOutputRows::BatchIndices( + batch.output_token_indices().to_vec(), + )); Ok(()) } @@ -493,7 +693,7 @@ impl<'model> LlamaContext<'model> { /// # Errors /// /// - `EncodeError` if the encoding failed. - pub fn encode(&mut self, batch: &mut LlamaBatch) -> Result<(), EncodeError> { + pub fn encode(&mut self, batch: &LlamaBatch) -> Result<(), EncodeError> { if self.exceeds_micro_batch(batch) { return Err(EncodeError::BatchExceedsMicroBatch { n_tokens: batch.n_tokens(), @@ -501,20 +701,23 @@ impl<'model> LlamaContext<'model> { }); } + self.transfer_batch(batch)?; + let mut out_llama_cpp_return_code: i32 = 0; let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_encode( self.context.as_ptr(), - batch.llama_batch, + self.batch_ext.as_ptr(), &raw mut out_llama_cpp_return_code, &raw mut out_error, ) }; encode_status_to_result(status, out_llama_cpp_return_code, out_error)?; - self.initialized_logits - .clone_from(&batch.initialized_logits); + self.record_decoded_rows(DecodedOutputRows::BatchIndices( + batch.output_token_indices().to_vec(), + )); Ok(()) } @@ -718,12 +921,17 @@ impl<'model> LlamaContext<'model> { Ok(LlamaTokenDataArray::from_iter(self.candidates()?, false)) } + fn produced_logits(&self) -> Result, LogitsError> { + NonNull::new(unsafe { llama_cpp_bindings_sys::llama_get_logits(self.context.as_ptr()) }) + .ok_or_else(|| self.decoded_outputs.missing_logits_error()) + } + /// # Errors - /// Returns `LogitsError` if the logits pointer is null or `n_vocab` overflows. + /// Returns `LogitsError` if the last decode produced no logits or `n_vocab` overflows. pub fn get_logits(&self) -> Result<&[f32], LogitsError> { - let data = unsafe { llama_cpp_bindings_sys::llama_get_logits(self.context.as_ptr()) }; + let logits = self.produced_logits()?; - unsafe { logits_slice_from_raw_parts(data, self.model.n_vocab()) } + unsafe { logits_slice(logits, self.model.n_vocab()) } } /// # Errors @@ -768,20 +976,22 @@ impl<'model> LlamaContext<'model> { Ok(()) } + /// Accepts the batch index of an output token, or a negative index counting back from the + /// last output. + /// /// # Errors - /// Returns `LogitsError` if the token is not initialized, out of range, or `n_vocab` overflows. + /// Returns `LogitsError` if the token is not an output of the last decode, out of range, the + /// last decode produced no logits, or `n_vocab` overflows. pub fn get_logits_ith(&self, token_index: i32) -> Result<&[f32], LogitsError> { - if !self.initialized_logits.contains(&token_index) { - return Err(LogitsError::TokenNotInitialized(token_index)); - } - - token_index_within_context(token_index, self.n_ctx())?; + self.decoded_outputs.ensure_output(token_index)?; + self.produced_logits()?; - let data = unsafe { + let row = NonNull::new(unsafe { llama_cpp_bindings_sys::llama_get_logits_ith(self.context.as_ptr(), token_index) - }; + }) + .ok_or(LogitsError::NullLogits)?; - unsafe { logits_slice_from_raw_parts(data, self.model.n_vocab()) } + unsafe { logits_slice(row, self.model.n_vocab()) } } pub fn reset_timings(&mut self) { @@ -789,8 +999,9 @@ impl<'model> LlamaContext<'model> { } pub fn timings(&mut self) -> LlamaTimings { - let timings = unsafe { llama_cpp_bindings_sys::llama_perf_context(self.context.as_ptr()) }; - LlamaTimings { timings } + LlamaTimings::from(unsafe { + llama_cpp_bindings_sys::llama_perf_context(self.context.as_ptr()) + }) } /// # Errors @@ -829,7 +1040,10 @@ impl<'model> LlamaContext<'model> { impl Drop for LlamaContext<'_> { fn drop(&mut self) { - unsafe { llama_cpp_bindings_sys::llama_free(self.context.as_ptr()) } + unsafe { + llama_cpp_bindings_sys::llama_batch_ext_free(self.batch_ext.as_ptr()); + llama_cpp_bindings_sys::llama_free(self.context.as_ptr()); + } } } @@ -1468,52 +1682,16 @@ mod encode_status_to_result_tests { } #[cfg(test)] -mod token_index_within_context_tests { - use super::token_index_within_context; - use crate::LogitsError; - - #[test] - fn token_index_beyond_context_size_maps_exceeds_context() { - let result = token_index_within_context(5, 4); - - assert_eq!( - result, - Err(LogitsError::TokenIndexExceedsContext { - token_index: 5, - context_size: 4, - }) - ); - } - - #[test] - fn token_index_within_context_size_is_ok() { - assert!(token_index_within_context(2, 4).is_ok()); - } - - #[test] - fn token_index_negative_skips_context_check() { - assert!(token_index_within_context(-1, 4).is_ok()); - } -} - -#[cfg(test)] -mod logits_slice_from_raw_parts_tests { - use std::ptr; +mod logits_slice_tests { + use std::ptr::NonNull; - use super::logits_slice_from_raw_parts; + use super::logits_slice; use crate::LogitsError; - #[test] - fn logits_slice_from_null_data_maps_null_logits() { - let result = unsafe { logits_slice_from_raw_parts(ptr::null(), 4) }; - - assert_eq!(result, Err(LogitsError::NullLogits)); - } - #[test] fn logits_slice_from_negative_vocab_maps_vocab_size_overflow() { let logit_value = 0.0_f32; - let result = unsafe { logits_slice_from_raw_parts(&raw const logit_value, -1) }; + let result = unsafe { logits_slice(NonNull::from(&logit_value), -1) }; let conversion_error = usize::try_from(-1_i32).unwrap_err(); diff --git a/llama-cpp-bindings/src/context/context_limits.rs b/llama-cpp-bindings/src/context/context_limits.rs new file mode 100644 index 000000000..adf7d0cb2 --- /dev/null +++ b/llama-cpp-bindings/src/context/context_limits.rs @@ -0,0 +1,152 @@ +use std::num::NonZeroU32; + +use llama_cpp_bindings_sys::llama_seq_id; + +use crate::context::LlamaContext; +use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; +use crate::error::token_outside_vocabulary::TokenOutsideVocabulary; +use crate::token::LlamaToken; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ContextLimits { + pub n_batch: u32, + pub n_ubatch: u32, + pub n_seq_max: u32, + pub n_vocab: i32, +} + +impl ContextLimits { + #[must_use] + pub fn of(context: &LlamaContext) -> Self { + Self { + n_batch: context.n_batch(), + n_ubatch: context.n_ubatch(), + n_seq_max: context.n_seq_max(), + n_vocab: context.model.n_vocab(), + } + } + + /// # Errors + /// + /// Returns [`TokenOutsideVocabulary`] when `token` is not in the vocabulary of the model. + pub fn ensure_token_in_vocabulary( + &self, + token: LlamaToken, + ) -> Result { + TokenOutsideVocabulary::check(token, self.n_vocab) + } + + /// # Errors + /// + /// Returns [`SequenceIdOutOfRange`] when `seq_id` names no sequence of the context. + pub fn ensure_sequence_id(&self, seq_id: llama_seq_id) -> Result<(), SequenceIdOutOfRange> { + if u32::try_from(seq_id).is_ok_and(|sequence| sequence < self.n_seq_max) { + Ok(()) + } else { + Err(SequenceIdOutOfRange { + seq_id, + n_seq_max: self.n_seq_max, + }) + } + } + + /// Returns how many tokens a single decode of `batch_tokens` evaluates: the smaller of + /// `batch_tokens` and the context's micro batch. + #[must_use] + pub const fn micro_batch_tokens(&self, batch_tokens: NonZeroU32) -> u32 { + let batch_tokens = batch_tokens.get(); + + if batch_tokens < self.n_ubatch { + batch_tokens + } else { + self.n_ubatch + } + } +} + +#[cfg(test)] +mod tests { + use std::num::NonZeroU32; + + use super::ContextLimits; + use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; + use crate::error::token_outside_vocabulary::TokenOutsideVocabulary; + use crate::token::LlamaToken; + + const LIMITS: ContextLimits = ContextLimits { + n_batch: 512, + n_ubatch: 128, + n_seq_max: 2, + n_vocab: 100, + }; + + #[test] + fn the_last_sequence_of_the_context_is_accepted() { + assert_eq!(LIMITS.ensure_sequence_id(1), Ok(())); + } + + #[test] + fn a_negative_sequence_id_is_rejected() { + assert_eq!( + LIMITS.ensure_sequence_id(-1), + Err(SequenceIdOutOfRange { + seq_id: -1, + n_seq_max: 2, + }) + ); + } + + #[test] + fn a_sequence_id_past_the_last_sequence_is_rejected() { + assert_eq!( + LIMITS.ensure_sequence_id(2), + Err(SequenceIdOutOfRange { + seq_id: 2, + n_seq_max: 2, + }) + ); + } + + #[test] + fn the_last_token_of_the_vocabulary_is_accepted() { + assert_eq!( + LIMITS.ensure_token_in_vocabulary(LlamaToken::new(99)), + Ok(LlamaToken::new(99)) + ); + } + + #[test] + fn a_token_past_the_vocabulary_is_rejected() { + assert_eq!( + LIMITS.ensure_token_in_vocabulary(LlamaToken::new(100)), + Err(TokenOutsideVocabulary { + token: LlamaToken::new(100), + n_vocab: 100, + }) + ); + } + + #[test] + fn a_negative_token_is_rejected() { + assert_eq!( + LIMITS.ensure_token_in_vocabulary(LlamaToken::new(-1)), + Err(TokenOutsideVocabulary { + token: LlamaToken::new(-1), + n_vocab: 100, + }) + ); + } + + #[test] + fn a_batch_smaller_than_the_micro_batch_is_decoded_whole() { + assert_eq!(LIMITS.micro_batch_tokens(NonZeroU32::new(64).unwrap()), 64); + } + + #[test] + fn a_batch_larger_than_the_micro_batch_is_decoded_in_micro_batches() { + assert_eq!( + LIMITS.micro_batch_tokens(NonZeroU32::new(256).unwrap()), + 128 + ); + } +} diff --git a/llama-cpp-bindings/src/context/decoded_output_rows.rs b/llama-cpp-bindings/src/context/decoded_output_rows.rs new file mode 100644 index 000000000..ffe792668 --- /dev/null +++ b/llama-cpp-bindings/src/context/decoded_output_rows.rs @@ -0,0 +1,39 @@ +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum DecodedOutputRows { + BatchIndices(Vec), + LastRow, +} + +impl DecodedOutputRows { + /// The rows a llama.cpp helper writes when it decodes on its own batches, which output at most + /// the last row and only when asked to. + #[must_use] + pub const fn of_helper_decode(logits_last: bool) -> Self { + if logits_last { + Self::LastRow + } else { + Self::BatchIndices(Vec::new()) + } + } +} + +#[cfg(test)] +mod tests { + use super::DecodedOutputRows; + + #[test] + fn a_helper_decode_with_logits_last_outputs_its_last_row() { + assert_eq!( + DecodedOutputRows::of_helper_decode(true), + DecodedOutputRows::LastRow + ); + } + + #[test] + fn a_helper_decode_without_logits_last_outputs_no_row() { + assert_eq!( + DecodedOutputRows::of_helper_decode(false), + DecodedOutputRows::BatchIndices(Vec::new()) + ); + } +} diff --git a/llama-cpp-bindings/src/context/decoded_outputs.rs b/llama-cpp-bindings/src/context/decoded_outputs.rs new file mode 100644 index 000000000..376f2c77d --- /dev/null +++ b/llama-cpp-bindings/src/context/decoded_outputs.rs @@ -0,0 +1,135 @@ +use crate::LogitsError; +use crate::context::decoded_output_rows::DecodedOutputRows; +use crate::context::llama_pooling_type::LlamaPoolingType; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DecodedOutputs { + pub rows: DecodedOutputRows, + pub pooling_type: LlamaPoolingType, +} + +impl DecodedOutputs { + /// Accepts the batch index of an output token, or a negative index counting back from the + /// last output, the way `llama_get_logits_ith` resolves it. + /// + /// # Errors + /// + /// Returns [`LogitsError::TokenNotInitialized`] when the index names no output of the last + /// decode. + pub fn ensure_output(&self, token_index: i32) -> Result<(), LogitsError> { + let names_an_output = match &self.rows { + DecodedOutputRows::BatchIndices(output_token_indices) if token_index >= 0 => { + output_token_indices.contains(&token_index) + } + DecodedOutputRows::BatchIndices(output_token_indices) => { + token_index.unsigned_abs() as usize <= output_token_indices.len() + } + DecodedOutputRows::LastRow => token_index == -1, + }; + + if names_an_output { + Ok(()) + } else { + Err(LogitsError::TokenNotInitialized(token_index)) + } + } + + #[must_use] + pub const fn missing_logits_error(&self) -> LogitsError { + match self.pooling_type { + LlamaPoolingType::None | LlamaPoolingType::Unspecified => { + LogitsError::GraphProducesNoLogits + } + pooling_type @ (LlamaPoolingType::Mean + | LlamaPoolingType::Cls + | LlamaPoolingType::Last + | LlamaPoolingType::Rank) => { + LogitsError::PooledEmbeddingsProduceNoLogits { pooling_type } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::DecodedOutputs; + use crate::LogitsError; + use crate::context::decoded_output_rows::DecodedOutputRows; + use crate::context::llama_pooling_type::LlamaPoolingType; + + fn outputs_at(output_token_indices: Vec) -> DecodedOutputs { + DecodedOutputs { + rows: DecodedOutputRows::BatchIndices(output_token_indices), + pooling_type: LlamaPoolingType::None, + } + } + + fn last_row_only() -> DecodedOutputs { + DecodedOutputs { + rows: DecodedOutputRows::LastRow, + pooling_type: LlamaPoolingType::None, + } + } + + #[test] + fn a_batch_index_marked_as_output_names_an_output() { + assert_eq!(outputs_at(vec![2, 5]).ensure_output(5), Ok(())); + } + + #[test] + fn a_batch_index_not_marked_as_output_is_refused() { + assert_eq!( + outputs_at(vec![2, 5]).ensure_output(3), + Err(LogitsError::TokenNotInitialized(3)) + ); + } + + #[test] + fn a_negative_index_counts_back_from_the_last_output() { + assert_eq!(outputs_at(vec![2, 5]).ensure_output(-2), Ok(())); + } + + #[test] + fn a_negative_index_beyond_the_first_output_is_refused() { + assert_eq!( + outputs_at(vec![2, 5]).ensure_output(-3), + Err(LogitsError::TokenNotInitialized(-3)) + ); + } + + #[test] + fn a_decode_outputting_only_its_last_row_names_it_from_the_end() { + assert_eq!(last_row_only().ensure_output(-1), Ok(())); + } + + #[test] + fn a_decode_outputting_only_its_last_row_refuses_batch_indices() { + assert_eq!( + last_row_only().ensure_output(0), + Err(LogitsError::TokenNotInitialized(0)) + ); + } + + #[test] + fn missing_logits_of_a_pooled_decode_are_attributed_to_pooling() { + let decoded = DecodedOutputs { + rows: DecodedOutputRows::BatchIndices(vec![0]), + pooling_type: LlamaPoolingType::Mean, + }; + + assert_eq!( + decoded.missing_logits_error(), + LogitsError::PooledEmbeddingsProduceNoLogits { + pooling_type: LlamaPoolingType::Mean + } + ); + } + + #[test] + fn missing_logits_of_an_unpooled_decode_are_attributed_to_the_graph() { + assert_eq!( + outputs_at(vec![0]).missing_logits_error(), + LogitsError::GraphProducesNoLogits + ); + } +} diff --git a/llama-cpp-bindings/src/context/params.rs b/llama-cpp-bindings/src/context/params.rs index 1538dc16c..9ec231939 100644 --- a/llama-cpp-bindings/src/context/params.rs +++ b/llama-cpp-bindings/src/context/params.rs @@ -217,6 +217,19 @@ impl LlamaContextParams { KvCacheType::from(self.context_params.type_v) } + /// Bytes of device memory caching the experts kept in host memory, split among the devices + /// like the layers; `0` disables the cache. + #[must_use] + pub const fn with_moe_cache_size(mut self, moe_cache_size: usize) -> Self { + self.context_params.moe_cache_size = moe_cache_size; + self + } + + #[must_use] + pub const fn moe_cache_size(&self) -> usize { + self.context_params.moe_cache_size + } + #[must_use] pub fn with_attention_type(mut self, attention_type: LlamaAttentionType) -> Self { self.context_params.attention_type = i32::from(attention_type); @@ -518,6 +531,13 @@ mod tests { assert_eq!(params.type_v(), KvCacheType::Q4_1); } + #[test] + fn with_moe_cache_size_sets_value() { + let params = super::LlamaContextParams::default().with_moe_cache_size(1 << 30); + + assert_eq!(params.moe_cache_size(), 1 << 30); + } + #[test] fn with_flash_attention_policy_sets_value() { let params = super::LlamaContextParams::default() diff --git a/llama-cpp-bindings/src/dry_sequence_breakers.rs b/llama-cpp-bindings/src/dry_sequence_breakers.rs new file mode 100644 index 000000000..640ca0403 --- /dev/null +++ b/llama-cpp-bindings/src/dry_sequence_breakers.rs @@ -0,0 +1,139 @@ +use std::collections::HashSet; +use std::ffi::CString; + +use llama_cpp_bindings_sys::LLAMA_DRY_MAX_SEQ_BREAKERS; + +use crate::SamplingError; + +const DRY_SEQUENCE_BREAKER_MAX_BYTES: usize = 40; + +fn validated_dry_sequence_breaker( + index: usize, + seq_breaker: &[u8], +) -> Result { + if seq_breaker.is_empty() { + return Err(SamplingError::DrySequenceBreakerEmpty { index }); + } + + if seq_breaker.len() > DRY_SEQUENCE_BREAKER_MAX_BYTES { + return Err(SamplingError::DrySequenceBreakerTooLong { + index, + length: seq_breaker.len(), + max_length: DRY_SEQUENCE_BREAKER_MAX_BYTES, + }); + } + + CString::new(seq_breaker) + .map_err(|source| SamplingError::DrySequenceBreakerContainsNul { index, source }) +} + +/// Validates DRY sequence breakers against the limits `llama_sampler_init_dry` enforces by +/// silently skipping, truncating, or ignoring breakers. +/// +/// # Errors +/// +/// Returns the [`SamplingError`] variant describing the first breaker llama.cpp would skip, +/// truncate, or reject, or [`SamplingError::TooManyDistinctDrySequenceBreakers`] when more +/// distinct breakers are given than llama.cpp uses. +pub fn dry_sequence_breakers( + seq_breakers: impl IntoIterator>, +) -> Result, SamplingError> { + let validated = seq_breakers + .into_iter() + .enumerate() + .map(|(index, seq_breaker)| validated_dry_sequence_breaker(index, seq_breaker.as_ref())) + .collect::, SamplingError>>()?; + let distinct = validated.iter().collect::>().len(); + let max = LLAMA_DRY_MAX_SEQ_BREAKERS as usize; + + if distinct > max { + return Err(SamplingError::TooManyDistinctDrySequenceBreakers { distinct, max }); + } + + Ok(validated) +} + +#[cfg(test)] +mod tests { + use std::ffi::CString; + + use super::dry_sequence_breakers; + use crate::SamplingError; + + fn distinct_breakers(count: usize) -> Vec { + (0..count) + .map(|breaker_index| format!("breaker{breaker_index}")) + .collect() + } + + #[test] + fn breakers_within_every_limit_are_passed_through_in_order() { + assert_eq!( + dry_sequence_breakers(["\n", ":"]), + Ok(vec![c"\n".to_owned(), c":".to_owned()]) + ); + } + + #[test] + fn an_empty_breaker_is_refused_instead_of_skipped() { + assert_eq!( + dry_sequence_breakers(["\n", ""]), + Err(SamplingError::DrySequenceBreakerEmpty { index: 1 }) + ); + } + + #[test] + fn a_breaker_longer_than_forty_bytes_is_refused_instead_of_truncated() { + assert_eq!( + dry_sequence_breakers(["a".repeat(41)]), + Err(SamplingError::DrySequenceBreakerTooLong { + index: 0, + length: 41, + max_length: 40, + }) + ); + } + + #[test] + fn a_breaker_of_exactly_forty_bytes_is_accepted() { + assert_eq!( + dry_sequence_breakers(["a".repeat(40)]), + Ok(vec![CString::new("a".repeat(40)).unwrap()]) + ); + } + + #[test] + fn a_breaker_with_an_interior_nul_byte_is_refused() { + let nul_error = CString::new("hello\0world").unwrap_err(); + + assert_eq!( + dry_sequence_breakers(["ok", "hello\0world"]), + Err(SamplingError::DrySequenceBreakerContainsNul { + index: 1, + source: nul_error, + }) + ); + } + + #[test] + fn sixty_four_distinct_breakers_with_duplicates_are_accepted() { + let mut breakers = distinct_breakers(64); + breakers.push("breaker0".to_owned()); + + assert_eq!( + dry_sequence_breakers(&breakers).map(|validated| validated.len()), + Ok(65) + ); + } + + #[test] + fn more_than_sixty_four_distinct_breakers_are_refused_instead_of_ignored() { + assert_eq!( + dry_sequence_breakers(distinct_breakers(65)), + Err(SamplingError::TooManyDistinctDrySequenceBreakers { + distinct: 65, + max: 64, + }) + ); + } +} diff --git a/llama-cpp-bindings/src/error.rs b/llama-cpp-bindings/src/error.rs index ce43614b0..80c851c0d 100644 --- a/llama-cpp-bindings/src/error.rs +++ b/llama-cpp-bindings/src/error.rs @@ -1,4 +1,5 @@ pub mod apply_chat_template_error; +pub mod batch_transfer_error; pub mod bracketed_args_failure; pub mod chat_template_error; pub mod clear_kv_cache_seq_error; @@ -32,6 +33,7 @@ pub mod sample_error; pub mod sampler_accept_error; pub mod sampler_apply_error; pub mod sampling_error; +pub mod sequence_id_out_of_range; pub mod string_to_token_error; pub mod token_attr_error; pub mod token_outside_vocabulary; diff --git a/llama-cpp-bindings/src/error/batch_transfer_error.rs b/llama-cpp-bindings/src/error/batch_transfer_error.rs new file mode 100644 index 000000000..eb100b69b --- /dev/null +++ b/llama-cpp-bindings/src/error/batch_transfer_error.rs @@ -0,0 +1,20 @@ +use crate::context::context_limits::ContextLimits; + +#[derive(Debug, Eq, PartialEq, thiserror::Error)] +pub enum BatchTransferError { + #[error(transparent)] + FfiStatus(#[from] crate::FfiStatusError), + #[error(transparent)] + FfiContract(#[from] crate::FfiContractError), + #[error("the batch was built for context limits {batch:?}, but the context has {context:?}")] + LimitsMismatch { + batch: ContextLimits, + context: ContextLimits, + }, + #[error("not enough memory")] + NotEnoughMemory, + #[error("the llama.cpp library ran out of memory")] + LlamaCppOutOfMemory, + #[error("{message}")] + Reported { message: String }, +} diff --git a/llama-cpp-bindings/src/error/decode_error.rs b/llama-cpp-bindings/src/error/decode_error.rs index 0eb296a64..ad41b9710 100644 --- a/llama-cpp-bindings/src/error/decode_error.rs +++ b/llama-cpp-bindings/src/error/decode_error.rs @@ -1,6 +1,8 @@ use std::num::NonZeroI32; use std::os::raw::c_int; +use crate::error::batch_transfer_error::BatchTransferError; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum DecodeError { #[error(transparent)] @@ -25,6 +27,8 @@ pub enum DecodeError { LlamaCppOutOfMemory, #[error("decode batch of {n_tokens} tokens must fit a single micro batch of {n_ubatch} tokens")] BatchExceedsMicroBatch { n_tokens: i32, n_ubatch: u32 }, + #[error("the batch could not be handed to llama.cpp: {0}")] + BatchTransfer(#[from] BatchTransferError), #[error("{message}")] Reported { message: String }, } diff --git a/llama-cpp-bindings/src/error/encode_error.rs b/llama-cpp-bindings/src/error/encode_error.rs index 3238c040d..4b0d605c6 100644 --- a/llama-cpp-bindings/src/error/encode_error.rs +++ b/llama-cpp-bindings/src/error/encode_error.rs @@ -1,6 +1,8 @@ use std::num::NonZeroI32; use std::os::raw::c_int; +use crate::error::batch_transfer_error::BatchTransferError; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum EncodeError { #[error(transparent)] @@ -25,6 +27,8 @@ pub enum EncodeError { LlamaCppOutOfMemory, #[error("encode batch of {n_tokens} tokens must fit a single micro batch of {n_ubatch} tokens")] BatchExceedsMicroBatch { n_tokens: i32, n_ubatch: u32 }, + #[error("the batch could not be handed to llama.cpp: {0}")] + BatchTransfer(#[from] BatchTransferError), #[error("{message}")] Reported { message: String }, } diff --git a/llama-cpp-bindings/src/error/fit_error.rs b/llama-cpp-bindings/src/error/fit_error.rs index 8aa5a00e9..3e6f35c38 100644 --- a/llama-cpp-bindings/src/error/fit_error.rs +++ b/llama-cpp-bindings/src/error/fit_error.rs @@ -8,6 +8,8 @@ pub enum FitError { NoFittingMemoryLayout, #[error("parameter fitting aborted")] Aborted, + #[error("{actual} memory margins were given, one per device needs exactly {expected}")] + MarginsDeviceCountMismatch { expected: usize, actual: usize }, #[error("parameter fitting returned an unknown status code: {code}")] UnknownStatus { code: i32 }, #[error("not enough memory")] diff --git a/llama-cpp-bindings/src/error/grammar_error.rs b/llama-cpp-bindings/src/error/grammar_error.rs index e092d0053..bd273ef0a 100644 --- a/llama-cpp-bindings/src/error/grammar_error.rs +++ b/llama-cpp-bindings/src/error/grammar_error.rs @@ -23,8 +23,6 @@ pub enum GrammarError { RootContainsNul(#[source] NulError), #[error("a lazy-grammar trigger pattern contains an interior NUL byte")] TriggerPatternContainsNul(#[source] NulError), - #[error("a DRY sequence breaker contains an interior NUL byte")] - SequenceBreakerContainsNul(#[source] NulError), #[error("the llguidance parser factory could not be created: {message}")] LlguidanceFactoryUnavailable { message: String }, #[error("the llguidance grammar could not be parsed: {message}")] diff --git a/llama-cpp-bindings/src/error/json_schema_to_grammar_error.rs b/llama-cpp-bindings/src/error/json_schema_to_grammar_error.rs index 73c5373dc..aed9048af 100644 --- a/llama-cpp-bindings/src/error/json_schema_to_grammar_error.rs +++ b/llama-cpp-bindings/src/error/json_schema_to_grammar_error.rs @@ -11,6 +11,8 @@ pub enum JsonSchemaToGrammarError { SchemaContainsNulByte(#[from] NulError), #[error("JSON schema is invalid: {message}")] InvalidSchema { message: String }, + #[error("JSON schema is not valid JSON: {message}")] + InvalidJson { message: String }, #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/error/logits_error.rs b/llama-cpp-bindings/src/error/logits_error.rs index 8462a9b9e..f5ee5d815 100644 --- a/llama-cpp-bindings/src/error/logits_error.rs +++ b/llama-cpp-bindings/src/error/logits_error.rs @@ -1,13 +1,17 @@ +use crate::context::llama_pooling_type::LlamaPoolingType; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum LogitsError { #[error("logits data pointer is null")] NullLogits, + #[error( + "the last decode pooled embeddings with {pooling_type:?} pooling, which produces no logits" + )] + PooledEmbeddingsProduceNoLogits { pooling_type: LlamaPoolingType }, + #[error("the model graph of the last decode produces no logits")] + GraphProducesNoLogits, #[error("logit for token index {0} is not initialized")] TokenNotInitialized(i32), - #[error("token index {token_index} exceeds context size {context_size}")] - TokenIndexExceedsContext { token_index: u32, context_size: u32 }, #[error("n_vocab does not fit into usize: {0}")] VocabSizeOverflow(#[source] std::num::TryFromIntError), - #[error("token_index does not fit into u32: {0}")] - TokenIndexOverflow(#[source] std::num::TryFromIntError), } diff --git a/llama-cpp-bindings/src/error/sample_error.rs b/llama-cpp-bindings/src/error/sample_error.rs index 37d28de86..21c554d91 100644 --- a/llama-cpp-bindings/src/error/sample_error.rs +++ b/llama-cpp-bindings/src/error/sample_error.rs @@ -1,3 +1,4 @@ +use crate::error::logits_error::LogitsError; use crate::error::sampler_apply_error::SamplerApplyError; use crate::error::token_to_string_error::TokenToStringError; @@ -7,6 +8,8 @@ pub enum SampleError { FfiStatus(#[from] crate::FfiStatusError), #[error(transparent)] FfiContract(#[from] crate::FfiContractError), + #[error("the logits to sample from are unavailable: {0}")] + Logits(#[from] LogitsError), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/error/sampling_error.rs b/llama-cpp-bindings/src/error/sampling_error.rs index 8f7fb8ac5..4f6718799 100644 --- a/llama-cpp-bindings/src/error/sampling_error.rs +++ b/llama-cpp-bindings/src/error/sampling_error.rs @@ -1,3 +1,5 @@ +use std::ffi::NulError; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum SamplingError { #[error(transparent)] @@ -17,4 +19,24 @@ pub enum SamplingError { }, #[error("{sampler} sampler could not be initialized")] SamplerUnavailable { sampler: &'static str }, + #[error("DRY sequence breaker {index} is empty")] + DrySequenceBreakerEmpty { index: usize }, + #[error( + "DRY sequence breaker {index} is {length} bytes long, more than the {max_length} bytes llama.cpp uses" + )] + DrySequenceBreakerTooLong { + index: usize, + length: usize, + max_length: usize, + }, + #[error("DRY sequence breaker {index} contains an interior NUL byte")] + DrySequenceBreakerContainsNul { + index: usize, + #[source] + source: NulError, + }, + #[error( + "{distinct} distinct DRY sequence breakers were given, more than the {max} llama.cpp uses" + )] + TooManyDistinctDrySequenceBreakers { distinct: usize, max: usize }, } diff --git a/llama-cpp-bindings/src/error/sequence_id_out_of_range.rs b/llama-cpp-bindings/src/error/sequence_id_out_of_range.rs new file mode 100644 index 000000000..f23f40676 --- /dev/null +++ b/llama-cpp-bindings/src/error/sequence_id_out_of_range.rs @@ -0,0 +1,6 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +#[error("sequence id {seq_id} names no sequence of a context with {n_seq_max} sequences")] +pub struct SequenceIdOutOfRange { + pub seq_id: i32, + pub n_seq_max: u32, +} diff --git a/llama-cpp-bindings/src/error/token_outside_vocabulary.rs b/llama-cpp-bindings/src/error/token_outside_vocabulary.rs index 6598c8b06..553c7d86e 100644 --- a/llama-cpp-bindings/src/error/token_outside_vocabulary.rs +++ b/llama-cpp-bindings/src/error/token_outside_vocabulary.rs @@ -6,3 +6,17 @@ pub struct TokenOutsideVocabulary { pub token: LlamaToken, pub n_vocab: i32, } + +impl TokenOutsideVocabulary { + /// # Errors + /// + /// Returns [`TokenOutsideVocabulary`] when `token` is not one of the `n_vocab` tokens of the + /// vocabulary. + pub fn check(token: LlamaToken, n_vocab: i32) -> Result { + if (0..n_vocab).contains(&token.0) { + Ok(token) + } else { + Err(Self { token, n_vocab }) + } + } +} diff --git a/llama-cpp-bindings/src/json_schema_to_grammar.rs b/llama-cpp-bindings/src/json_schema_to_grammar.rs index c4cc7615c..3eed3495f 100644 --- a/llama-cpp-bindings/src/json_schema_to_grammar.rs +++ b/llama-cpp-bindings/src/json_schema_to_grammar.rs @@ -43,6 +43,16 @@ unsafe fn json_schema_to_grammar_status_to_result( }?; Err(JsonSchemaToGrammarError::InvalidSchema { message }) } + llama_cpp_bindings_sys::LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_INVALID_JSON => { + let message = unsafe { + read_and_free_cpp_string( + error_ptr, + "llama_rs_json_schema_to_grammar", + "reported a thrown C++ exception without an error message", + ) + }?; + Err(JsonSchemaToGrammarError::InvalidJson { message }) + } llama_cpp_bindings_sys::LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_LLAMA_CPP_THREW_CXX_EXCEPTION => { let message = unsafe { read_and_free_cpp_string( @@ -94,7 +104,6 @@ pub fn json_schema_to_grammar(schema_json: &str) -> Result Result { - let incremented = n_tokens - .checked_add(1) - .ok_or(BatchAddError::TokenCountOverflow { n_tokens })?; - - checked_i32_as_usize(incremented, "the incremented token count") -} - -fn checked_i32_as_usize( - value: i32, - value_description: &'static str, -) -> Result { - usize::try_from(value).map_err(|source| BatchAddError::IntegerOverflow { - value_description, - target_type: "usize", - source, - }) -} - -fn checked_usize_as_llama_seq_id( - value: usize, - value_description: &'static str, -) -> Result { - llama_seq_id::try_from(value).map_err(|source| BatchAddError::IntegerOverflow { - value_description, - target_type: "llama_seq_id", - source, - }) -} - -fn checked_usize_as_i32( - value: usize, - value_description: &'static str, -) -> Result { - i32::try_from(value).map_err(|source| BatchAddError::IntegerOverflow { - value_description, - target_type: "i32", - source, - }) +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LlamaBatch { + limits: ContextLimits, + entries: Vec, + output_token_indices: Vec, + n_tokens: i32, } -fn checked_usize_as_llama_pos( - value: usize, - value_description: &'static str, -) -> Result { - llama_pos::try_from(value).map_err(|source| BatchAddError::IntegerOverflow { - value_description, - target_type: "llama_pos", - source, - }) -} - -#[derive(Debug)] -pub struct LlamaBatch<'tokens> { - allocated: usize, - pub initialized_logits: Vec, - pub llama_batch: llama_batch, - n_seq_max: i32, - phantom: PhantomData<&'tokens [LlamaToken]>, -} +impl LlamaBatch { + #[must_use] + pub const fn new(limits: ContextLimits) -> Self { + Self { + limits, + entries: Vec::new(), + output_token_indices: Vec::new(), + n_tokens: 0, + } + } -impl<'tokens> LlamaBatch<'tokens> { pub fn clear(&mut self) { - self.llama_batch.n_tokens = 0; - self.initialized_logits.clear(); + self.entries.clear(); + self.output_token_indices.clear(); + self.n_tokens = 0; } /// # Errors /// - /// Returns an error if there is insufficient space in the buffer, if the token belongs to more - /// sequences than the batch was created for, or if integer conversions fail. + /// Returns [`BatchAddError`] when the batch already holds as many tokens as one decode of the + /// context takes, the token is outside the vocabulary, `seq_ids` is empty or names a sequence + /// the context does not have, or the token count would overflow. pub fn add( &mut self, sampled_token: &SampledToken, pos: llama_pos, - seq_ids: &[i32], + seq_ids: &[llama_seq_id], logits: bool, ) -> Result<(), BatchAddError> { - let (SampledToken::Content(LlamaToken(id)) - | SampledToken::Reasoning(LlamaToken(id)) - | SampledToken::ToolCall(LlamaToken(id)) - | SampledToken::Undeterminable(LlamaToken(id))) = *sampled_token; - let required = checked_n_tokens_plus_one_as_usize(self.n_tokens())?; - - if self.allocated < required { - return Err(BatchAddError::InsufficientSpace(self.allocated)); + if self.n_tokens.unsigned_abs() >= self.limits.n_batch { + return Err(BatchAddError::BatchFull { + n_batch: self.limits.n_batch, + }); } - let offset = self.llama_batch.n_tokens; - let offset_usize = checked_i32_as_usize(offset, "n_tokens")?; - let n_seq_id = checked_usize_as_llama_seq_id(seq_ids.len(), "seq_ids.len()")?; + let token = self + .limits + .ensure_token_in_vocabulary(sampled_token.token())?; - if n_seq_id > self.n_seq_max { - return Err(BatchAddError::TooManySequenceIds { - sequence_ids: n_seq_id, - n_seq_max: self.n_seq_max, - }); + if seq_ids.is_empty() { + return Err(BatchAddError::NoSequenceIds); } - unsafe { - self.llama_batch.token.add(offset_usize).write(id); - self.llama_batch.pos.add(offset_usize).write(pos); - self.llama_batch.n_seq_id.add(offset_usize).write(n_seq_id); - for (seq_index, seq_id) in seq_ids.iter().enumerate() { - let tmp = *self.llama_batch.seq_id.add(offset_usize); - tmp.add(seq_index).write(*seq_id); - } - self.llama_batch - .logits - .add(offset_usize) - .write(i8::from(logits)); + for seq_id in seq_ids { + self.limits.ensure_sequence_id(*seq_id)?; } + let token_index = self.n_tokens; + self.n_tokens = token_index + .checked_add(1) + .ok_or(BatchAddError::TokenCountOverflow { + n_tokens: token_index, + })?; + if logits { - self.initialized_logits.push(offset); + self.output_token_indices.push(token_index); } - self.llama_batch.n_tokens += 1; + self.entries.push(LlamaBatchEntry { + token, + pos, + seq_ids: seq_ids.to_vec(), + output: logits, + }); Ok(()) } + /// Adds `tokens` at positions counting from zero in `seq_id`, with logits for the last one, + /// or for every one when `logits_all` is set. + /// /// # Errors /// - /// Returns an error if there is insufficient space in the buffer or if integer conversions fail. + /// Returns the [`BatchAddError`] of the first token [`Self::add`] refuses. pub fn add_sequence( &mut self, tokens: &[LlamaToken], - seq_id: i32, + seq_id: llama_seq_id, logits_all: bool, ) -> Result<(), BatchAddError> { - let last_index = checked_usize_as_llama_pos(tokens.len().saturating_sub(1), "n_tokens")?; - - for (position, token) in (0..).zip(tokens.iter()) { + for ((token_index, token), pos) in tokens.iter().enumerate().zip(0..) { self.add( &SampledToken::Content(*token), - position, + pos, &[seq_id], - logits_all || position == last_index, + logits_all || token_index + 1 == tokens.len(), )?; } @@ -156,462 +107,178 @@ impl<'tokens> LlamaBatch<'tokens> { } #[must_use] - pub fn new(capacity: BatchCapacity, n_seq_max: i32) -> Self { - let batch = unsafe { llama_batch_init(capacity.tokens_i32(), 0, n_seq_max) }; - - LlamaBatch { - allocated: capacity.tokens_usize(), - initialized_logits: vec![], - llama_batch: batch, - n_seq_max, - phantom: PhantomData, - } + pub const fn limits(&self) -> ContextLimits { + self.limits } - /// # Errors - /// - /// Returns an error if the provided token buffer is empty or if integer conversions fail. - pub fn get_one(tokens: &'tokens [LlamaToken]) -> Result { - if tokens.is_empty() { - return Err(BatchAddError::EmptyBuffer); - } - - let token_count = checked_usize_as_i32(tokens.len(), "token count")?; - - let batch = unsafe { - let ptr = tokens.as_ptr().cast::().cast_mut(); - llama_cpp_bindings_sys::llama_batch_get_one(ptr, token_count) - }; - - let last_token_index = checked_usize_as_i32(tokens.len() - 1, "last token index")?; - - Ok(Self { - allocated: 0, - initialized_logits: vec![last_token_index], - llama_batch: batch, - n_seq_max: 0, - phantom: PhantomData, - }) + #[must_use] + pub const fn n_tokens(&self) -> i32 { + self.n_tokens } #[must_use] - pub const fn n_tokens(&self) -> i32 { - self.llama_batch.n_tokens + pub fn entries(&self) -> &[LlamaBatchEntry] { + &self.entries } -} -impl Drop for LlamaBatch<'_> { - fn drop(&mut self) { - unsafe { - if self.allocated > 0 { - llama_batch_free(self.llama_batch); - } - } + #[must_use] + pub fn output_token_indices(&self) -> &[i32] { + &self.output_token_indices } } #[cfg(test)] mod tests { - use llama_cpp_bindings_types::BatchCapacity; - + use crate::batch_add_error::BatchAddError; + use crate::context::context_limits::ContextLimits; + use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; + use crate::error::token_outside_vocabulary::TokenOutsideVocabulary; + use crate::llama_batch_entry::LlamaBatchEntry; use crate::sampled_token::SampledToken; use crate::token::LlamaToken; - use super::{ - BatchAddError, LlamaBatch, checked_i32_as_usize, checked_n_tokens_plus_one_as_usize, - checked_usize_as_i32, checked_usize_as_llama_pos, checked_usize_as_llama_seq_id, - }; - - #[test] - fn new_creates_empty_batch() { - let batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); - - assert_eq!(batch.n_tokens(), 0); - assert_eq!(batch.initialized_logits, Vec::::new()); - } - - #[test] - fn clear_resets_batch() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); - batch - .add(&SampledToken::Content(LlamaToken::new(1)), 0, &[0], true) - .unwrap(); - assert_eq!(batch.n_tokens(), 1); - - batch.clear(); - - assert_eq!(batch.n_tokens(), 0); - assert_eq!(batch.initialized_logits, Vec::::new()); - } - - #[test] - fn add_increments_token_count() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); - - batch - .add(&SampledToken::Content(LlamaToken::new(1)), 0, &[0], false) - .unwrap(); - assert_eq!(batch.n_tokens(), 1); + use super::LlamaBatch; - batch - .add(&SampledToken::Content(LlamaToken::new(2)), 1, &[0], false) - .unwrap(); - assert_eq!(batch.n_tokens(), 2); - } - - #[test] - fn add_tracks_logits() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); - - batch - .add(&SampledToken::Content(LlamaToken::new(1)), 0, &[0], false) - .unwrap(); - assert_eq!(batch.initialized_logits, Vec::::new()); + const LIMITS: ContextLimits = ContextLimits { + n_batch: 4, + n_ubatch: 4, + n_seq_max: 2, + n_vocab: 100, + }; - batch - .add(&SampledToken::Content(LlamaToken::new(2)), 1, &[0], true) - .unwrap(); - assert_eq!(batch.initialized_logits, vec![1]); + fn content(token_id: i32) -> SampledToken { + SampledToken::Content(LlamaToken::new(token_id)) } #[test] - fn add_returns_insufficient_space_when_full() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(1).unwrap(), 1); - batch - .add(&SampledToken::Content(LlamaToken::new(1)), 0, &[0], false) - .unwrap(); + fn added_tokens_are_kept_in_order_with_their_outputs() { + let mut batch = LlamaBatch::new(LIMITS); - let result = batch.add(&SampledToken::Content(LlamaToken::new(2)), 1, &[0], false); - - assert_eq!(result, Err(BatchAddError::InsufficientSpace(1))); - } - - #[test] - fn add_refuses_more_sequence_ids_than_the_batch_holds_per_token() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); - - let result = batch.add( - &SampledToken::Content(LlamaToken::new(1)), - 0, - &[0, 1], - false, - ); + batch.add(&content(1), 0, &[0], false).unwrap(); + batch.add(&content(2), 1, &[0, 1], true).unwrap(); assert_eq!( - result, - Err(BatchAddError::TooManySequenceIds { - sequence_ids: 2, - n_seq_max: 1, - }) + batch.entries(), + [ + LlamaBatchEntry { + token: LlamaToken::new(1), + pos: 0, + seq_ids: vec![0], + output: false, + }, + LlamaBatchEntry { + token: LlamaToken::new(2), + pos: 1, + seq_ids: vec![0, 1], + output: true, + }, + ] ); - assert_eq!(batch.n_tokens(), 0); - } - - #[test] - fn add_accepts_reasoning_sampled_token_variant() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(4).unwrap(), 1); - - batch - .add(&SampledToken::Reasoning(LlamaToken::new(11)), 0, &[0], true) - .unwrap(); - - assert_eq!(batch.n_tokens(), 1); - } - - #[test] - fn add_accepts_tool_call_sampled_token_variant() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(4).unwrap(), 1); - - batch - .add(&SampledToken::ToolCall(LlamaToken::new(22)), 0, &[0], true) - .unwrap(); - - assert_eq!(batch.n_tokens(), 1); - } - - #[test] - fn add_accepts_undeterminable_sampled_token_variant() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(4).unwrap(), 1); - - batch - .add( - &SampledToken::Undeterminable(LlamaToken::new(33)), - 0, - &[0], - false, - ) - .unwrap(); - - assert_eq!(batch.n_tokens(), 1); - } - - #[test] - fn add_sequence_adds_all_tokens() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); - let tokens = vec![ - LlamaToken::new(10), - LlamaToken::new(20), - LlamaToken::new(30), - ]; - - batch.add_sequence(&tokens, 0, false).unwrap(); - - assert_eq!(batch.n_tokens(), 3); - } - - #[test] - fn add_sequence_sets_logits_on_last_token() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); - let tokens = vec![ - LlamaToken::new(10), - LlamaToken::new(20), - LlamaToken::new(30), - ]; - - batch.add_sequence(&tokens, 0, false).unwrap(); - - assert_eq!(batch.initialized_logits, vec![2]); - } - - #[test] - fn add_sequence_insufficient_space() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2).unwrap(), 1); - let tokens = vec![ - LlamaToken::new(10), - LlamaToken::new(20), - LlamaToken::new(30), - ]; - - let result = batch.add_sequence(&tokens, 0, false); - - assert!(result.is_err()); - } - - #[test] - fn add_sequence_fails_mid_loop_when_batch_fills() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2).unwrap(), 1); - batch - .add(&SampledToken::Content(LlamaToken::new(1)), 0, &[0], false) - .unwrap(); - - let tokens = vec![LlamaToken::new(10), LlamaToken::new(20)]; - let result = batch.add_sequence(&tokens, 0, false); - - assert!(result.is_err()); - } - - #[test] - fn get_one_with_valid_tokens() { - let tokens = vec![LlamaToken::new(1), LlamaToken::new(2)]; - let batch = LlamaBatch::get_one(&tokens).expect("test: get_one should succeed"); - + assert_eq!(batch.output_token_indices(), [1]); assert_eq!(batch.n_tokens(), 2); - assert_eq!(batch.initialized_logits, vec![1]); } #[test] - fn get_one_empty_slice_returns_error() { - let tokens: Vec = vec![]; - let result = LlamaBatch::get_one(&tokens); - - assert_eq!(result.unwrap_err(), BatchAddError::EmptyBuffer); - } + fn clearing_forgets_every_token_and_output() { + let mut batch = LlamaBatch::new(LIMITS); + batch.add(&content(1), 0, &[0], true).unwrap(); - #[test] - fn get_one_single_token() { - let tokens = vec![LlamaToken::new(42)]; - let batch = LlamaBatch::get_one(&tokens).expect("test: get_one should succeed"); + batch.clear(); - assert_eq!(batch.n_tokens(), 1); - assert_eq!(batch.initialized_logits, vec![0]); + assert_eq!(batch, LlamaBatch::new(LIMITS)); } #[test] - fn add_with_logits_false_retains_only_previous_logits() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); - - batch - .add(&SampledToken::Content(LlamaToken::new(1)), 0, &[0], true) - .unwrap(); - assert_eq!(batch.initialized_logits, vec![0]); - + fn a_batch_holding_one_decode_of_tokens_refuses_another() { + let mut batch = LlamaBatch::new(LIMITS); batch - .add(&SampledToken::Content(LlamaToken::new(2)), 0, &[0], false) + .add_sequence(&[1, 2, 3, 4].map(LlamaToken::new), 0, false) .unwrap(); - assert_eq!(batch.initialized_logits, vec![0]); - } - - #[test] - fn add_sequence_with_logits_all_marks_every_token() -> Result<(), BatchAddError> { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); - let tokens = vec![ - LlamaToken::new(10), - LlamaToken::new(20), - LlamaToken::new(30), - ]; - - batch.add_sequence(&tokens, 0, true)?; - - assert_eq!(batch.n_tokens(), 3); - assert_eq!(batch.initialized_logits, vec![0, 1, 2]); - - Ok(()) - } - - #[test] - fn add_with_multiple_seq_ids() -> Result<(), BatchAddError> { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 4); - - batch.add( - &SampledToken::Content(LlamaToken::new(1)), - 0, - &[0, 1, 2], - true, - )?; - - assert_eq!(batch.n_tokens(), 1); - assert_eq!(batch.initialized_logits, vec![0]); - - Ok(()) - } - - #[test] - fn drop_does_not_free_get_one_batch() { - let tokens = vec![LlamaToken::new(1), LlamaToken::new(2)]; - let batch = LlamaBatch::get_one(&tokens).expect("test: get_one should succeed"); - - assert_eq!(batch.allocated, 0); - drop(batch); - } - #[test] - fn checked_n_tokens_plus_one_as_usize_succeeds_for_zero() { - let result = checked_n_tokens_plus_one_as_usize(0); - - assert_eq!(result, Ok(1)); - } - - #[test] - fn checked_n_tokens_plus_one_as_usize_fails_for_negative() { - assert_eq!( - checked_n_tokens_plus_one_as_usize(-2), - Err(BatchAddError::IntegerOverflow { - value_description: "the incremented token count", - target_type: "usize", - source: usize::try_from(-1_i32).expect_err("the conversion must fail"), - }) - ); - } - - #[test] - fn checked_n_tokens_plus_one_as_usize_fails_for_i32_max() { assert_eq!( - checked_n_tokens_plus_one_as_usize(i32::MAX), - Err(BatchAddError::TokenCountOverflow { n_tokens: i32::MAX }) + batch.add(&content(5), 4, &[0], false), + Err(BatchAddError::BatchFull { n_batch: 4 }) ); } #[test] - fn checked_i32_as_usize_succeeds_for_zero() { - let result = checked_i32_as_usize(0, "test_value"); + fn a_token_outside_the_vocabulary_is_refused() { + let mut batch = LlamaBatch::new(LIMITS); - assert_eq!(result, Ok(0)); - } - - #[test] - fn checked_i32_as_usize_fails_for_negative() { assert_eq!( - checked_i32_as_usize(i32::MIN, "test_value"), - Err(BatchAddError::IntegerOverflow { - value_description: "test_value", - target_type: "usize", - source: usize::try_from(i32::MIN).expect_err("the conversion must fail"), - }) + batch.add(&content(100), 0, &[0], false), + Err(BatchAddError::TokenOutsideVocabulary( + TokenOutsideVocabulary { + token: LlamaToken::new(100), + n_vocab: 100, + } + )) ); } #[test] - fn checked_usize_as_llama_seq_id_succeeds_for_zero() { - let result = checked_usize_as_llama_seq_id(0, "test_value"); - - assert_eq!(result, Ok(0)); - } + fn a_token_without_sequences_is_refused() { + let mut batch = LlamaBatch::new(LIMITS); - #[test] - fn checked_usize_as_llama_seq_id_fails_for_overflow() { assert_eq!( - checked_usize_as_llama_seq_id(usize::MAX, "test_value"), - Err(BatchAddError::IntegerOverflow { - value_description: "test_value", - target_type: "llama_seq_id", - source: super::llama_seq_id::try_from(usize::MAX) - .expect_err("the conversion must fail"), - }) + batch.add(&content(1), 0, &[], false), + Err(BatchAddError::NoSequenceIds) ); } #[test] - fn checked_usize_as_i32_succeeds_for_zero() { - let result = checked_usize_as_i32(0, "test_value"); + fn a_token_in_a_sequence_the_context_lacks_is_refused_and_not_added() { + let mut batch = LlamaBatch::new(LIMITS); - assert_eq!(result, Ok(0)); - } - - #[test] - fn checked_usize_as_i32_fails_for_overflow() { assert_eq!( - checked_usize_as_i32(usize::MAX, "test_value"), - Err(BatchAddError::IntegerOverflow { - value_description: "test_value", - target_type: "i32", - source: i32::try_from(usize::MAX).expect_err("the conversion must fail"), - }) + batch.add(&content(1), 0, &[0, 2], false), + Err(BatchAddError::SequenceIdOutOfRange(SequenceIdOutOfRange { + seq_id: 2, + n_seq_max: 2, + })) ); + assert_eq!(batch, LlamaBatch::new(LIMITS)); } #[test] - fn checked_usize_as_llama_pos_succeeds_for_zero() { - let result = checked_usize_as_llama_pos(0, "test_value"); + fn a_sequence_requests_logits_for_its_last_token_only() { + let mut batch = LlamaBatch::new(LIMITS); - assert_eq!(result, Ok(0)); - } + batch + .add_sequence(&[10, 20, 30].map(LlamaToken::new), 1, false) + .unwrap(); - #[test] - fn checked_usize_as_llama_pos_fails_for_overflow() { + assert_eq!(batch.output_token_indices(), [2]); assert_eq!( - checked_usize_as_llama_pos(usize::MAX, "test_value"), - Err(BatchAddError::IntegerOverflow { - value_description: "test_value", - target_type: "llama_pos", - source: super::llama_pos::try_from(usize::MAX) - .expect_err("the conversion must fail"), + batch.entries().last(), + Some(&LlamaBatchEntry { + token: LlamaToken::new(30), + pos: 2, + seq_ids: vec![1], + output: true, }) ); } #[test] - fn add_fails_when_required_token_count_overflows_i32() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); - batch.llama_batch.n_tokens = i32::MAX; + fn a_sequence_with_logits_for_all_requests_every_token() { + let mut batch = LlamaBatch::new(LIMITS); - assert_eq!( - batch.add(&SampledToken::Content(LlamaToken::new(1)), 0, &[0], false), - Err(BatchAddError::TokenCountOverflow { n_tokens: i32::MAX }) - ); + batch + .add_sequence(&[10, 20, 30].map(LlamaToken::new), 0, true) + .unwrap(); + + assert_eq!(batch.output_token_indices(), [0, 1, 2]); } #[test] - fn add_fails_when_existing_offset_is_negative() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); - batch.llama_batch.n_tokens = -1; + fn a_sequence_longer_than_one_decode_stops_at_the_first_refused_token() { + let mut batch = LlamaBatch::new(LIMITS); assert_eq!( - batch.add(&SampledToken::Content(LlamaToken::new(1)), 0, &[0], false), - Err(BatchAddError::IntegerOverflow { - value_description: "n_tokens", - target_type: "usize", - source: usize::try_from(-1_i32).expect_err("the conversion must fail"), - }) + batch.add_sequence(&[1, 2, 3, 4, 5].map(LlamaToken::new), 0, false), + Err(BatchAddError::BatchFull { n_batch: 4 }) ); + assert_eq!(batch.n_tokens(), 4); } } diff --git a/llama-cpp-bindings/src/llama_batch_entry.rs b/llama-cpp-bindings/src/llama_batch_entry.rs new file mode 100644 index 000000000..1a73063e8 --- /dev/null +++ b/llama-cpp-bindings/src/llama_batch_entry.rs @@ -0,0 +1,11 @@ +use llama_cpp_bindings_sys::{llama_pos, llama_seq_id}; + +use crate::token::LlamaToken; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LlamaBatchEntry { + pub token: LlamaToken, + pub pos: llama_pos, + pub seq_ids: Vec, + pub output: bool, +} diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index 04105da10..513bb9fa5 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -419,13 +419,7 @@ impl LlamaModel { &self, token: LlamaToken, ) -> Result { - let n_vocab = self.n_vocab(); - - if (0..n_vocab).contains(&token.0) { - Ok(token) - } else { - Err(TokenOutsideVocabulary { token, n_vocab }) - } + TokenOutsideVocabulary::check(token, self.n_vocab()) } #[must_use] diff --git a/llama-cpp-bindings/src/model/params.rs b/llama-cpp-bindings/src/model/params.rs index 8265a7e96..fe742373c 100644 --- a/llama-cpp-bindings/src/model/params.rs +++ b/llama-cpp-bindings/src/model/params.rs @@ -416,6 +416,13 @@ impl LlamaModelParams { let max_devices = unsafe { llama_cpp_bindings_sys::llama_max_devices() }; let max_buft = unsafe { llama_cpp_bindings_sys::llama_max_tensor_buft_overrides() }; + if margins.len() != max_devices { + return Err(FitError::MarginsDeviceCountMismatch { + expected: max_devices, + actual: margins.len(), + }); + } + self.tensor_split.clear(); self.tensor_split.resize(max_devices, 0.0); @@ -901,6 +908,33 @@ mod tests { ); } + #[test] + fn fit_params_refuses_margins_not_sized_to_the_device_count() { + use crate::context::params::LlamaContextParams; + use crate::error::FitError; + + let mut params = std::pin::pin!(LlamaModelParams::default()); + let mut context_params = LlamaContextParams::default(); + let mut margins = vec![0usize; crate::max_devices() + 1]; + + let result = params.as_mut().fit_params( + c"/nonexistent/path/to/model.gguf", + &mut context_params, + &mut margins, + 512, + None, + llama_cpp_bindings_sys::GGML_LOG_LEVEL_NONE, + ); + + assert_eq!( + result.err(), + Some(FitError::MarginsDeviceCountMismatch { + expected: crate::max_devices(), + actual: crate::max_devices() + 1, + }) + ); + } + #[test] fn fit_params_status_to_result_maps_null_path_model_arg() { assert_eq!( diff --git a/llama-cpp-bindings/src/model/vocab_type.rs b/llama-cpp-bindings/src/model/vocab_type.rs index 14e151322..eba208403 100644 --- a/llama-cpp-bindings/src/model/vocab_type.rs +++ b/llama-cpp-bindings/src/model/vocab_type.rs @@ -3,8 +3,15 @@ use crate::model::vocab_type_from_int_error::VocabTypeFromIntError; #[repr(u32)] #[derive(Debug, Eq, Copy, Clone, PartialEq)] pub enum VocabType { - BPE = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_BPE as _, - SPM = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_SPM as _, + None = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_NONE as _, + Spm = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_SPM as _, + Bpe = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_BPE as _, + Wpm = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_WPM as _, + Ugm = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_UGM as _, + Rwkv = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_RWKV as _, + Plamo2 = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_PLAMO2 as _, + Test = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_TEST as _, + Plamo3 = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_PLAMO3 as _, } impl TryFrom for VocabType { @@ -12,8 +19,15 @@ impl TryFrom for VocabType { fn try_from(value: llama_cpp_bindings_sys::llama_vocab_type) -> Result { match value { - llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_BPE => Ok(Self::BPE), - llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_SPM => Ok(Self::SPM), + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_NONE => Ok(Self::None), + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_SPM => Ok(Self::Spm), + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_BPE => Ok(Self::Bpe), + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_WPM => Ok(Self::Wpm), + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_UGM => Ok(Self::Ugm), + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_RWKV => Ok(Self::Rwkv), + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_PLAMO2 => Ok(Self::Plamo2), + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_TEST => Ok(Self::Test), + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_PLAMO3 => Ok(Self::Plamo3), unknown => Err(VocabTypeFromIntError::UnknownValue(unknown)), } } @@ -24,21 +38,38 @@ mod tests { use super::{VocabType, VocabTypeFromIntError}; #[test] - fn try_from_bpe() { - let result = VocabType::try_from(llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_BPE); + fn every_llama_cpp_vocab_type_maps_to_its_variant() { + let mapped = [ + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_NONE, + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_SPM, + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_BPE, + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_WPM, + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_UGM, + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_RWKV, + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_PLAMO2, + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_TEST, + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_PLAMO3, + ] + .map(VocabType::try_from); - assert_eq!(result, Ok(VocabType::BPE)); + assert_eq!( + mapped, + [ + Ok(VocabType::None), + Ok(VocabType::Spm), + Ok(VocabType::Bpe), + Ok(VocabType::Wpm), + Ok(VocabType::Ugm), + Ok(VocabType::Rwkv), + Ok(VocabType::Plamo2), + Ok(VocabType::Test), + Ok(VocabType::Plamo3), + ] + ); } #[test] - fn try_from_spm() { - let result = VocabType::try_from(llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_SPM); - - assert_eq!(result, Ok(VocabType::SPM)); - } - - #[test] - fn try_from_unknown_value() { + fn a_vocab_type_unknown_to_llama_cpp_is_refused() { let result = VocabType::try_from(99999); assert_eq!(result, Err(VocabTypeFromIntError::UnknownValue(99999))); diff --git a/llama-cpp-bindings/src/mtmd.rs b/llama-cpp-bindings/src/mtmd.rs index 2d7c4212e..6db9270f0 100644 --- a/llama-cpp-bindings/src/mtmd.rs +++ b/llama-cpp-bindings/src/mtmd.rs @@ -2,15 +2,16 @@ mod mtmd_ffi_input_part; pub mod fitted_mtmd_input_chunk; pub mod fitted_mtmd_input_chunks; -pub mod micro_batch_tokens; pub mod mtmd_bitmap; pub mod mtmd_bitmap_error; +pub mod mtmd_capability; pub mod mtmd_context; pub mod mtmd_context_params; pub mod mtmd_default_marker; pub mod mtmd_default_marker_error; pub mod mtmd_encode_error; pub mod mtmd_eval_error; +pub mod mtmd_eval_limits; pub mod mtmd_init_error; pub mod mtmd_input_chunk; pub mod mtmd_input_chunk_error; @@ -23,19 +24,19 @@ pub mod mtmd_input_text; pub mod mtmd_input_text_part; pub mod mtmd_tokenize_error; pub mod non_causal_chunk_micro_batch_mismatch; -pub mod positive_batch_tokens; pub use fitted_mtmd_input_chunk::FittedMtmdInputChunk; pub use fitted_mtmd_input_chunks::FittedMtmdInputChunks; -pub use micro_batch_tokens::micro_batch_tokens; pub use mtmd_bitmap::MtmdBitmap; pub use mtmd_bitmap_error::MtmdBitmapError; +pub use mtmd_capability::MtmdCapability; pub use mtmd_context::MtmdContext; pub use mtmd_context_params::MtmdContextParams; pub use mtmd_default_marker::mtmd_default_marker; pub use mtmd_default_marker_error::MtmdDefaultMarkerError; pub use mtmd_encode_error::MtmdEncodeError; pub use mtmd_eval_error::MtmdEvalError; +pub use mtmd_eval_limits::MtmdEvalLimits; pub use mtmd_init_error::MtmdInitError; pub use mtmd_input_chunk::MtmdInputChunk; pub use mtmd_input_chunk_error::MtmdInputChunkError; diff --git a/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs b/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs index abdd1f4bf..7895ede72 100644 --- a/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs @@ -3,17 +3,17 @@ use std::ffi::c_char; use std::marker::PhantomData; use crate::context::LlamaContext; +use crate::context::decoded_output_rows::DecodedOutputRows; use crate::token::LlamaToken; use llama_cpp_ffi_status::read_and_free_cpp_string; use super::fitted_mtmd_input_chunks::FittedMtmdInputChunks; -use super::micro_batch_tokens::micro_batch_tokens; use super::mtmd_context::MtmdContext; use super::mtmd_eval_error::MtmdEvalError; +use super::mtmd_eval_limits::MtmdEvalLimits; use super::mtmd_input_chunk::MtmdInputChunk; use super::mtmd_input_chunk_type::MtmdInputChunkType; use super::mtmd_input_chunks::MtmdInputChunks; -use super::positive_batch_tokens::positive_batch_tokens; fn eval_chunk_single_status_to_result( status: llama_cpp_bindings_sys::llama_rs_mtmd_eval_chunk_single_status, @@ -126,21 +126,23 @@ impl<'fitted> FittedMtmdInputChunk<'fitted> { /// # Errors /// - /// Returns [`MtmdEvalError::NonPositiveBatchSize`] when `n_batch` is not positive, + /// Returns any error of [`MtmdEvalLimits::batch_tokens`], /// [`MtmdEvalError::EvaluatedMicroBatchSmallerThanFitted`] when decodes of `n_batch` in /// `llama_ctx` are smaller than the micro batch the chunks were fitted to, or /// [`MtmdEvalError::EvalFailed`] if the underlying encode or decode step fails. pub fn eval_single( &self, mtmd_ctx: &MtmdContext, - llama_ctx: &LlamaContext, + llama_ctx: &mut LlamaContext, start_position: llama_cpp_bindings_sys::llama_pos, seq_id: llama_cpp_bindings_sys::llama_seq_id, n_batch: i32, logits_last: bool, ) -> Result { - let evaluated_micro_batch_tokens = - micro_batch_tokens(llama_ctx, positive_batch_tokens(n_batch)?); + let limits = MtmdEvalLimits::of(llama_ctx); + let evaluated_micro_batch_tokens = limits + .context + .micro_batch_tokens(limits.batch_tokens(n_batch, seq_id)?); if evaluated_micro_batch_tokens < self.micro_batch_tokens { return Err(MtmdEvalError::EvaluatedMicroBatchSmallerThanFitted { @@ -168,12 +170,15 @@ impl<'fitted> FittedMtmdInputChunk<'fitted> { ) }; - eval_chunk_single_status_to_result( + let next_position = eval_chunk_single_status_to_result( status, final_position, out_llama_cpp_return_code, out_error, - ) + )?; + llama_ctx.record_decoded_rows(DecodedOutputRows::of_helper_decode(logits_last)); + + Ok(next_position) } } diff --git a/llama-cpp-bindings/src/mtmd/micro_batch_tokens.rs b/llama-cpp-bindings/src/mtmd/micro_batch_tokens.rs deleted file mode 100644 index 3fb97599d..000000000 --- a/llama-cpp-bindings/src/mtmd/micro_batch_tokens.rs +++ /dev/null @@ -1,10 +0,0 @@ -use std::num::NonZeroU32; - -use crate::context::LlamaContext; - -/// Returns how many tokens a single decode of `n_batch` evaluates in `llama_ctx`: the smaller of -/// `n_batch` and the context's micro batch. -#[must_use] -pub fn micro_batch_tokens(llama_ctx: &LlamaContext, n_batch: NonZeroU32) -> u32 { - n_batch.get().min(llama_ctx.n_ubatch()) -} diff --git a/llama-cpp-bindings/src/mtmd/mtmd_capability.rs b/llama-cpp-bindings/src/mtmd/mtmd_capability.rs new file mode 100644 index 000000000..ac8e1d7a9 --- /dev/null +++ b/llama-cpp-bindings/src/mtmd/mtmd_capability.rs @@ -0,0 +1,15 @@ +use enumflags2::bitflags; + +#[derive(Eq, PartialEq, Debug, Clone, Copy)] +#[bitflags] +#[repr(u8)] +pub enum MtmdCapability { + Chat = 1 << 0, + Vision = 1 << 1, + Audio = 1 << 2, + Video = 1 << 3, + AudioStream = 1 << 4, + TextGeneration = 1 << 5, + AudioGeneration = 1 << 6, + AudioGenerationSpeakerReference = 1 << 7, +} diff --git a/llama-cpp-bindings/src/mtmd/mtmd_context.rs b/llama-cpp-bindings/src/mtmd/mtmd_context.rs index 5c85500ff..6edd8a45f 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_context.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_context.rs @@ -4,11 +4,13 @@ use std::path::PathBuf; use std::ptr; use std::ptr::NonNull; +use enumflags2::BitFlags; use llama_cpp_ffi_status::read_and_free_cpp_string; use crate::model::LlamaModel; use super::mtmd_bitmap::MtmdBitmap; +use super::mtmd_capability::MtmdCapability; use super::mtmd_context_params::MtmdContextParams; use super::mtmd_encode_error::MtmdEncodeError; use super::mtmd_ffi_input_part::MtmdFfiInputPart; @@ -19,6 +21,37 @@ use super::mtmd_input_part::MtmdInputPart; use super::mtmd_input_text::MtmdInputText; use super::mtmd_tokenize_error::MtmdTokenizeError; +fn capability_flags( + llama_cpp_bindings_sys::mtmd_helper_cap { + chat, + vision, + audio, + video, + audio_stream, + text_gen, + audio_gen, + audio_gen_spk_ref, + }: llama_cpp_bindings_sys::mtmd_helper_cap, +) -> BitFlags { + [ + (chat, MtmdCapability::Chat), + (vision, MtmdCapability::Vision), + (audio, MtmdCapability::Audio), + (video, MtmdCapability::Video), + (audio_stream, MtmdCapability::AudioStream), + (text_gen, MtmdCapability::TextGeneration), + (audio_gen, MtmdCapability::AudioGeneration), + ( + audio_gen_spk_ref, + MtmdCapability::AudioGenerationSpeakerReference, + ), + ] + .into_iter() + .filter(|(is_capable, _capability)| *is_capable) + .map(|(_is_capable, capability)| capability) + .collect() +} + fn map_tokenize_status( status: llama_cpp_bindings_sys::llama_rs_mtmd_tokenize_status, undocumented_return_code: i32, @@ -319,13 +352,13 @@ impl MtmdContext { } #[must_use] - pub fn support_vision(&self) -> bool { - unsafe { llama_cpp_bindings_sys::mtmd_support_vision(self.context.as_ptr()) } - } - - #[must_use] - pub fn support_audio(&self) -> bool { - unsafe { llama_cpp_bindings_sys::mtmd_support_audio(self.context.as_ptr()) } + pub fn capabilities(&self, model: &LlamaModel) -> BitFlags { + capability_flags(unsafe { + llama_cpp_bindings_sys::mtmd_helper_get_capabilities( + model.model.as_ptr(), + self.context.as_ptr(), + ) + }) } #[must_use] @@ -1101,3 +1134,61 @@ mod map_init_from_file_status_tests { assert_eq!(outcome.err(), Some(MtmdInitError::LlamaCppOutOfMemory)); } } + +#[cfg(test)] +mod capability_flags_tests { + use enumflags2::BitFlags; + + use super::capability_flags; + use crate::mtmd::mtmd_capability::MtmdCapability; + + const NO_CAPABILITIES: llama_cpp_bindings_sys::mtmd_helper_cap = + llama_cpp_bindings_sys::mtmd_helper_cap { + chat: false, + vision: false, + audio: false, + video: false, + audio_stream: false, + text_gen: false, + audio_gen: false, + audio_gen_spk_ref: false, + }; + + #[test] + fn a_model_without_capabilities_has_no_flags() { + assert_eq!(capability_flags(NO_CAPABILITIES), BitFlags::empty()); + } + + #[test] + fn every_capability_of_llama_cpp_maps_to_its_own_flag() { + let all_capabilities = llama_cpp_bindings_sys::mtmd_helper_cap { + chat: true, + vision: true, + audio: true, + video: true, + audio_stream: true, + text_gen: true, + audio_gen: true, + audio_gen_spk_ref: true, + }; + + assert_eq!( + capability_flags(all_capabilities), + BitFlags::::all() + ); + } + + #[test] + fn a_text_to_speech_model_generates_audio_but_cannot_chat() { + let text_to_speech = llama_cpp_bindings_sys::mtmd_helper_cap { + audio_gen: true, + audio_gen_spk_ref: true, + ..NO_CAPABILITIES + }; + + assert_eq!( + capability_flags(text_to_speech), + MtmdCapability::AudioGeneration | MtmdCapability::AudioGenerationSpeakerReference + ); + } +} diff --git a/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs index 5f5663068..b6824a2b3 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs @@ -1,3 +1,4 @@ +use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; use crate::mtmd::mtmd_input_chunk_type_error::MtmdInputChunkTypeError; use crate::mtmd::non_causal_chunk_micro_batch_mismatch::NonCausalChunkMicroBatchMismatch; @@ -11,6 +12,8 @@ pub enum MtmdEvalError { BatchSizeExceedsContextLimit { requested: i32, context_max: u32 }, #[error("batch size {requested} must be positive")] NonPositiveBatchSize { requested: i32 }, + #[error(transparent)] + SequenceIdOutOfRange(#[from] SequenceIdOutOfRange), #[error( "a chunk decoded non-causally has {} tokens but a single decode fits {}", .0.chunk_tokens, diff --git a/llama-cpp-bindings/src/mtmd/mtmd_eval_limits.rs b/llama-cpp-bindings/src/mtmd/mtmd_eval_limits.rs new file mode 100644 index 000000000..340deb384 --- /dev/null +++ b/llama-cpp-bindings/src/mtmd/mtmd_eval_limits.rs @@ -0,0 +1,115 @@ +use std::num::NonZeroU32; + +use crate::context::LlamaContext; +use crate::context::context_limits::ContextLimits; + +use super::mtmd_eval_error::MtmdEvalError; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct MtmdEvalLimits { + pub context: ContextLimits, +} + +impl MtmdEvalLimits { + #[must_use] + pub fn of(llama_ctx: &LlamaContext) -> Self { + Self { + context: ContextLimits::of(llama_ctx), + } + } + + /// Returns `n_batch` as a token count once both it and `seq_id` fit the context, so the mtmd + /// helper never adds a token its batch refuses. + /// + /// # Errors + /// + /// Returns [`MtmdEvalError::NonPositiveBatchSize`] when `n_batch` is not positive, + /// [`MtmdEvalError::BatchSizeExceedsContextLimit`] when it exceeds the context's batch size, + /// or [`MtmdEvalError::SequenceIdOutOfRange`] when `seq_id` names no sequence of the context. + pub fn batch_tokens( + &self, + n_batch: i32, + seq_id: llama_cpp_bindings_sys::llama_seq_id, + ) -> Result { + let batch_tokens = u32::try_from(n_batch) + .ok() + .and_then(NonZeroU32::new) + .ok_or(MtmdEvalError::NonPositiveBatchSize { requested: n_batch })?; + + if batch_tokens.get() > self.context.n_batch { + return Err(MtmdEvalError::BatchSizeExceedsContextLimit { + requested: n_batch, + context_max: self.context.n_batch, + }); + } + + self.context.ensure_sequence_id(seq_id)?; + + Ok(batch_tokens) + } +} + +#[cfg(test)] +mod tests { + use std::num::NonZeroU32; + + use super::MtmdEvalLimits; + use crate::context::context_limits::ContextLimits; + use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; + use crate::mtmd::mtmd_eval_error::MtmdEvalError; + + const LIMITS: MtmdEvalLimits = MtmdEvalLimits { + context: ContextLimits { + n_batch: 512, + n_ubatch: 128, + n_seq_max: 2, + n_vocab: 100, + }, + }; + + #[test] + fn a_batch_within_the_context_for_an_existing_sequence_is_accepted() { + assert_eq!( + LIMITS.batch_tokens(512, 1), + Ok(NonZeroU32::new(512).unwrap()) + ); + } + + #[test] + fn a_zero_batch_is_rejected() { + assert_eq!( + LIMITS.batch_tokens(0, 0), + Err(MtmdEvalError::NonPositiveBatchSize { requested: 0 }) + ); + } + + #[test] + fn a_negative_batch_is_rejected() { + assert_eq!( + LIMITS.batch_tokens(-1, 0), + Err(MtmdEvalError::NonPositiveBatchSize { requested: -1 }) + ); + } + + #[test] + fn a_batch_larger_than_the_context_batch_is_rejected() { + assert_eq!( + LIMITS.batch_tokens(513, 0), + Err(MtmdEvalError::BatchSizeExceedsContextLimit { + requested: 513, + context_max: 512, + }) + ); + } + + #[test] + fn a_sequence_id_the_context_lacks_is_rejected() { + assert_eq!( + LIMITS.batch_tokens(64, 2), + Err(MtmdEvalError::SequenceIdOutOfRange(SequenceIdOutOfRange { + seq_id: 2, + n_seq_max: 2, + })) + ); + } +} diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs index 113ada7f1..ace152a16 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs @@ -1,14 +1,14 @@ use std::ptr::NonNull; use crate::context::LlamaContext; +use crate::context::decoded_output_rows::DecodedOutputRows; -use super::micro_batch_tokens::micro_batch_tokens; use super::mtmd_context::MtmdContext; use super::mtmd_eval_error::MtmdEvalError; +use super::mtmd_eval_limits::MtmdEvalLimits; use super::mtmd_input_chunk::MtmdInputChunk; use super::mtmd_input_chunk_type::MtmdInputChunkType; use super::mtmd_input_chunks_error::MtmdInputChunksError; -use super::positive_batch_tokens::positive_batch_tokens; const fn check_eval_result(result: i32) -> Result<(), MtmdEvalError> { if result == 0 { @@ -104,30 +104,25 @@ impl MtmdInputChunks { /// /// # Errors /// - /// Returns [`MtmdEvalError::BatchSizeExceedsContextLimit`] when `n_batch` exceeds the - /// context's batch size, [`MtmdEvalError::NonPositiveBatchSize`] when it is not positive, - /// any error of [`Self::fit_to_micro_batch`], or [`MtmdEvalError::EvalFailed`] if any - /// encoding or decoding operation fails. + /// Returns any error of [`MtmdEvalLimits::batch_tokens`] or [`Self::fit_to_micro_batch`], or + /// [`MtmdEvalError::EvalFailed`] if any encoding or decoding operation fails. pub fn eval_chunks( &self, mtmd_ctx: &MtmdContext, - llama_ctx: &LlamaContext, + llama_ctx: &mut LlamaContext, start_position: llama_cpp_bindings_sys::llama_pos, seq_id: llama_cpp_bindings_sys::llama_seq_id, n_batch: i32, logits_last: bool, ) -> Result { - let batch_tokens = positive_batch_tokens(n_batch)?; - let context_max_batch = llama_ctx.n_batch(); - - if batch_tokens.get() > context_max_batch { - return Err(MtmdEvalError::BatchSizeExceedsContextLimit { - requested: n_batch, - context_max: context_max_batch, - }); - } + let limits = MtmdEvalLimits::of(llama_ctx); - self.fit_to_micro_batch(mtmd_ctx, micro_batch_tokens(llama_ctx, batch_tokens))?; + self.fit_to_micro_batch( + mtmd_ctx, + limits + .context + .micro_batch_tokens(limits.batch_tokens(n_batch, seq_id)?), + )?; let mut final_position: llama_cpp_bindings_sys::llama_pos = start_position; @@ -145,6 +140,7 @@ impl MtmdInputChunks { }; check_eval_result(result)?; + llama_ctx.record_decoded_rows(DecodedOutputRows::of_helper_decode(logits_last)); Ok(final_position) } diff --git a/llama-cpp-bindings/src/mtmd/positive_batch_tokens.rs b/llama-cpp-bindings/src/mtmd/positive_batch_tokens.rs deleted file mode 100644 index 060aa6bd1..000000000 --- a/llama-cpp-bindings/src/mtmd/positive_batch_tokens.rs +++ /dev/null @@ -1,37 +0,0 @@ -use std::num::NonZeroU32; - -use super::mtmd_eval_error::MtmdEvalError; - -/// Returns `n_batch` as a positive token count. -/// -/// # Errors -/// -/// Returns [`MtmdEvalError::NonPositiveBatchSize`] when `n_batch` is not positive. -pub fn positive_batch_tokens(n_batch: i32) -> Result { - u32::try_from(n_batch) - .ok() - .and_then(NonZeroU32::new) - .ok_or(MtmdEvalError::NonPositiveBatchSize { requested: n_batch }) -} - -#[cfg(test)] -mod tests { - use super::positive_batch_tokens; - use crate::mtmd::mtmd_eval_error::MtmdEvalError; - - #[test] - fn a_zero_batch_is_rejected() { - assert_eq!( - positive_batch_tokens(0), - Err(MtmdEvalError::NonPositiveBatchSize { requested: 0 }) - ); - } - - #[test] - fn a_negative_batch_is_rejected() { - assert_eq!( - positive_batch_tokens(-1), - Err(MtmdEvalError::NonPositiveBatchSize { requested: -1 }) - ); - } -} diff --git a/llama-cpp-bindings/src/sampled_token.rs b/llama-cpp-bindings/src/sampled_token.rs index 776ead808..67b91cf70 100644 --- a/llama-cpp-bindings/src/sampled_token.rs +++ b/llama-cpp-bindings/src/sampled_token.rs @@ -7,3 +7,37 @@ pub enum SampledToken { ToolCall(LlamaToken), Undeterminable(LlamaToken), } + +impl SampledToken { + #[must_use] + pub const fn token(&self) -> LlamaToken { + match *self { + Self::Content(token) + | Self::Reasoning(token) + | Self::ToolCall(token) + | Self::Undeterminable(token) => token, + } + } +} + +#[cfg(test)] +mod tests { + use super::SampledToken; + use crate::token::LlamaToken; + + #[test] + fn every_section_carries_the_sampled_token() { + let token = LlamaToken::new(42); + + assert_eq!( + [ + SampledToken::Content(token), + SampledToken::Reasoning(token), + SampledToken::ToolCall(token), + SampledToken::Undeterminable(token), + ] + .map(|sampled_token| sampled_token.token()), + [token; 4] + ); + } +} diff --git a/llama-cpp-bindings/src/sampled_token_classifier.rs b/llama-cpp-bindings/src/sampled_token_classifier.rs index 59fa49a75..0a2b9879c 100644 --- a/llama-cpp-bindings/src/sampled_token_classifier.rs +++ b/llama-cpp-bindings/src/sampled_token_classifier.rs @@ -23,8 +23,7 @@ use crate::mtmd::FittedMtmdInputChunks; use crate::mtmd::MtmdContext; use crate::mtmd::MtmdInputChunkType; use crate::mtmd::MtmdInputChunks; -use crate::mtmd::micro_batch_tokens; -use crate::mtmd::positive_batch_tokens::positive_batch_tokens; +use crate::mtmd::mtmd_eval_limits::MtmdEvalLimits; use crate::sampled_token::SampledToken; use crate::sampling::LlamaSampler; use crate::streaming_json_probe::StreamingJsonProbe; @@ -463,22 +462,25 @@ impl<'model> SampledTokenClassifier<'model> { /// them, so a chunk that cannot be decoded moves neither the KV cache nor the usage counters. /// /// # Errors - /// Returns [`EvalMultimodalChunksError::EvalFailed`] when `params.n_batch` is not positive, a - /// chunk cannot be decoded or reports a type unknown to this binding (all before any chunk is - /// evaluated), or when evaluating a chunk fails, or + /// Returns [`EvalMultimodalChunksError::EvalFailed`] when `params.n_batch` or `params.seq_id` + /// does not fit the context, a chunk cannot be decoded or reports a type unknown to this + /// binding (all before any chunk is evaluated), or when evaluating a chunk fails, or /// [`EvalMultimodalChunksError::ChunkOutOfBounds`] when a valid index returns /// `None` from `chunks.get`. pub fn eval_multimodal_chunks( &mut self, chunks: &MtmdInputChunks, mtmd_ctx: &MtmdContext, - llama_ctx: &LlamaContext, + llama_ctx: &mut LlamaContext, params: EvalMultimodalChunksParams, ) -> Result { + let limits = MtmdEvalLimits::of(llama_ctx); let fitted_chunks = FittedMtmdInputChunks::fit( chunks, mtmd_ctx, - micro_batch_tokens(llama_ctx, positive_batch_tokens(params.n_batch)?), + limits + .context + .micro_batch_tokens(limits.batch_tokens(params.n_batch, params.seq_id)?), )?; let chunk_count = fitted_chunks.len(); let mut next_position = params.start_position; diff --git a/llama-cpp-bindings/src/sampling.rs b/llama-cpp-bindings/src/sampling.rs index a950cc6f6..1a9ca147e 100644 --- a/llama-cpp-bindings/src/sampling.rs +++ b/llama-cpp-bindings/src/sampling.rs @@ -7,6 +7,7 @@ use llama_cpp_error_recorder::ErrorScope; use llama_cpp_error_recorder::RecordedError; use crate::context::LlamaContext; +use crate::dry_sequence_breakers::dry_sequence_breakers; use crate::model::LlamaModel; use crate::sanitized_grammar::SanitizedGrammar; use crate::token::LlamaToken; @@ -412,9 +413,12 @@ impl LlamaSampler { /// # Errors /// - /// Returns [`SampleError`] if the C++ sampler throws an exception, the index is invalid, or the - /// grammar sampler callback recorded a failure during sampling. + /// Returns [`SampleError::Logits`] when the context holds no logits for `idx`, or another + /// [`SampleError`] if the C++ sampler throws an exception or the grammar sampler callback + /// recorded a failure during sampling. pub fn sample(&mut self, ctx: &LlamaContext, idx: i32) -> Result { + ctx.get_logits_ith(idx)?; + let mut token: i32 = -1; let mut error_ptr: *mut c_char = std::ptr::null_mut(); @@ -771,7 +775,8 @@ impl LlamaSampler { } /// # Errors - /// Returns an error if any string in `seq_breakers` contains null bytes. + /// Returns the [`SamplingError`] describing a sequence breaker llama.cpp would skip, truncate, + /// or ignore, or the error of initializing the sampler. pub fn dry( model: &LlamaModel, multiplier: f32, @@ -779,13 +784,8 @@ impl LlamaSampler { allowed_length: i32, penalty_last_n: i32, seq_breakers: impl IntoIterator>, - ) -> Result { - let seq_breakers: Vec = seq_breakers - .into_iter() - .map(|seq_breaker| { - CString::new(seq_breaker.as_ref()).map_err(GrammarError::SequenceBreakerContainsNul) - }) - .collect::, _>>()?; + ) -> Result { + let seq_breakers = dry_sequence_breakers(seq_breakers)?; let mut seq_breaker_pointers: Vec<*const c_char> = seq_breakers .iter() .map(|seq_breaker| seq_breaker.as_ptr()) @@ -804,7 +804,6 @@ impl LlamaSampler { out_error, ) }) - .map_err(Into::into) } /// # Errors diff --git a/llama-cpp-bindings/src/timing.rs b/llama-cpp-bindings/src/timing.rs index ed68ec2fe..d42adaedf 100644 --- a/llama-cpp-bindings/src/timing.rs +++ b/llama-cpp-bindings/src/timing.rs @@ -1,5 +1,8 @@ -use std::fmt::{Debug, Display, Formatter}; -fn write_timings(timings: &LlamaTimings, writer: &mut dyn std::fmt::Write) -> std::fmt::Result { +use std::fmt::{Display, Formatter, Write}; + +use llama_cpp_bindings_sys::llama_perf_context_data; + +fn write_timings(timings: &LlamaTimings, writer: &mut dyn Write) -> std::fmt::Result { writeln!(writer, "load time = {:.2} ms", timings.t_load_ms())?; if timings.n_p_eval() > 0 { @@ -32,38 +35,29 @@ fn write_timings(timings: &LlamaTimings, writer: &mut dyn std::fmt::Write) -> st writeln!(writer, "eval time = {:.2} ms / 0 runs", timings.t_eval_ms())?; } + writeln!(writer, "graphs reused = {}", timings.n_reused())?; + writeln!( + writer, + "graph reallocs = {} with the same size / {} with a changed size", + timings.n_reallocs_same(), + timings.n_reallocs_diff() + )?; + Ok(()) } #[derive(Clone, Copy, Debug)] pub struct LlamaTimings { - pub timings: llama_cpp_bindings_sys::llama_perf_context_data, + timings: llama_perf_context_data, } -impl LlamaTimings { - #[must_use] - pub const fn new( - t_start_ms: f64, - t_load_ms: f64, - t_p_eval_ms: f64, - t_eval_ms: f64, - n_p_eval: i32, - n_eval: i32, - n_reused: i32, - ) -> Self { - Self { - timings: llama_cpp_bindings_sys::llama_perf_context_data { - t_start_ms, - t_load_ms, - t_p_eval_ms, - t_eval_ms, - n_p_eval, - n_eval, - n_reused, - }, - } +impl From for LlamaTimings { + fn from(timings: llama_perf_context_data) -> Self { + Self { timings } } +} +impl LlamaTimings { #[must_use] pub const fn t_start_ms(&self) -> f64 { self.timings.t_start_ms @@ -94,28 +88,19 @@ impl LlamaTimings { self.timings.n_eval } - pub const fn set_t_start_ms(&mut self, t_start_ms: f64) { - self.timings.t_start_ms = t_start_ms; - } - - pub const fn set_t_load_ms(&mut self, t_load_ms: f64) { - self.timings.t_load_ms = t_load_ms; - } - - pub const fn set_t_p_eval_ms(&mut self, t_p_eval_ms: f64) { - self.timings.t_p_eval_ms = t_p_eval_ms; - } - - pub const fn set_t_eval_ms(&mut self, t_eval_ms: f64) { - self.timings.t_eval_ms = t_eval_ms; + #[must_use] + pub const fn n_reused(&self) -> i32 { + self.timings.n_reused } - pub const fn set_n_p_eval(&mut self, n_p_eval: i32) { - self.timings.n_p_eval = n_p_eval; + #[must_use] + pub const fn n_reallocs_same(&self) -> i32 { + self.timings.n_reallocs_same } - pub const fn set_n_eval(&mut self, n_eval: i32) { - self.timings.n_eval = n_eval; + #[must_use] + pub const fn n_reallocs_diff(&self) -> i32 { + self.timings.n_reallocs_diff } } @@ -127,156 +112,99 @@ impl Display for LlamaTimings { #[cfg(test)] mod tests { - use super::LlamaTimings; + use std::fmt::Write; - #[test] - fn display_format_with_valid_counts() { - let timings = LlamaTimings::new(1.0, 2.0, 3.0, 4.0, 5, 6, 1); - let output = format!("{timings}"); - - assert!(output.contains("load time = 2.00 ms")); - assert!(output.contains("prompt eval time = 3.00 ms / 5 tokens")); - assert!(output.contains("eval time = 4.00 ms / 6 runs")); - } - - #[test] - fn display_format_handles_zero_eval_counts() { - let timings = LlamaTimings::new(0.0, 1.0, 2.0, 3.0, 0, 0, 0); - let output = format!("{timings}"); - - assert!(output.contains("load time = 1.00 ms")); - assert!(output.contains("prompt eval time = 2.00 ms / 0 tokens")); - assert!(output.contains("eval time = 3.00 ms / 0 runs")); - assert!(!output.contains("NaN")); - assert!(!output.contains("inf")); - } - - #[test] - fn set_t_start_ms() { - let mut timings = LlamaTimings::new(0.0, 0.0, 0.0, 0.0, 0, 0, 0); - - timings.set_t_start_ms(42.0); - - assert!((timings.t_start_ms() - 42.0).abs() < f64::EPSILON); - } - - #[test] - fn set_t_load_ms() { - let mut timings = LlamaTimings::new(0.0, 0.0, 0.0, 0.0, 0, 0, 0); + use llama_cpp_bindings_sys::llama_perf_context_data; - timings.set_t_load_ms(10.5); + use super::LlamaTimings; + use super::write_timings; - assert!((timings.t_load_ms() - 10.5).abs() < f64::EPSILON); + struct WriterFailingAfterWrites { + remaining_writes: usize, } - #[test] - fn set_t_p_eval_ms() { - let mut timings = LlamaTimings::new(0.0, 0.0, 0.0, 0.0, 0, 0, 0); - - timings.set_t_p_eval_ms(7.7); + impl Write for WriterFailingAfterWrites { + fn write_str(&mut self, _text: &str) -> std::fmt::Result { + if self.remaining_writes == 0 { + return Err(std::fmt::Error); + } + self.remaining_writes -= 1; - assert!((timings.t_p_eval_ms() - 7.7).abs() < f64::EPSILON); + Ok(()) + } } - #[test] - fn set_t_eval_ms() { - let mut timings = LlamaTimings::new(0.0, 0.0, 0.0, 0.0, 0, 0, 0); - - timings.set_t_eval_ms(3.3); - - assert!((timings.t_eval_ms() - 3.3).abs() < f64::EPSILON); + struct WriterCountingWrites { + writes: usize, } - #[test] - fn set_n_p_eval() { - let mut timings = LlamaTimings::new(0.0, 0.0, 0.0, 0.0, 0, 0, 0); + impl Write for WriterCountingWrites { + fn write_str(&mut self, _text: &str) -> std::fmt::Result { + self.writes += 1; - timings.set_n_p_eval(100); - - assert_eq!(timings.n_p_eval(), 100); + Ok(()) + } } - #[test] - fn set_n_eval() { - let mut timings = LlamaTimings::new(0.0, 0.0, 0.0, 0.0, 0, 0, 0); - - timings.set_n_eval(200); - - assert_eq!(timings.n_eval(), 200); + const fn timings_with_eval_counts(n_p_eval: i32, n_eval: i32) -> LlamaTimings { + LlamaTimings { + timings: llama_perf_context_data { + t_start_ms: 1.0, + t_load_ms: 2.0, + t_p_eval_ms: 3.0, + t_eval_ms: 4.0, + n_p_eval, + n_eval, + n_reused: 7, + n_reallocs_same: 8, + n_reallocs_diff: 9, + }, + } } #[test] - fn write_timings_propagates_writer_errors() { - struct FailingWriter; - - impl std::fmt::Write for FailingWriter { - fn write_str(&mut self, _text: &str) -> std::fmt::Result { - Err(std::fmt::Error) - } - } + fn display_reports_throughput_of_evaluated_prompt_and_generated_tokens() { + let output = timings_with_eval_counts(5, 6).to_string(); - let timings = LlamaTimings::new(1.0, 2.0, 3.0, 4.0, 5, 6, 1); - let result = super::write_timings(&timings, &mut FailingWriter); - - assert!(result.is_err()); + assert_eq!( + output, + "load time = 2.00 ms\n\ + prompt eval time = 3.00 ms / 5 tokens (0.60 ms per token, 1666.67 tokens per second)\n\ + eval time = 4.00 ms / 6 runs (0.67 ms per token, 1500.00 tokens per second)\n\ + graphs reused = 7\n\ + graph reallocs = 8 with the same size / 9 with a changed size\n" + ); } #[test] - fn write_timings_zero_p_eval_with_failing_writer() { - struct FailAfterNWrites { - remaining: usize, - } - - impl std::fmt::Write for FailAfterNWrites { - fn write_str(&mut self, _text: &str) -> std::fmt::Result { - if self.remaining == 0 { - return Err(std::fmt::Error); - } - self.remaining -= 1; - - Ok(()) - } - } + fn display_omits_throughput_when_nothing_was_evaluated() { + let output = timings_with_eval_counts(0, 0).to_string(); - let timings = LlamaTimings::new(1.0, 2.0, 3.0, 4.0, 0, 6, 1); - let result = super::write_timings(&timings, &mut FailAfterNWrites { remaining: 1 }); - - assert!(result.is_err()); + assert_eq!( + output, + "load time = 2.00 ms\n\ + prompt eval time = 3.00 ms / 0 tokens\n\ + eval time = 4.00 ms / 0 runs\n\ + graphs reused = 7\n\ + graph reallocs = 8 with the same size / 9 with a changed size\n" + ); } #[test] - fn write_timings_fails_at_each_writeln_boundary() { - struct FailAfterNWrites { - remaining: usize, - } - - impl std::fmt::Write for FailAfterNWrites { - fn write_str(&mut self, _text: &str) -> std::fmt::Result { - if self.remaining == 0 { - return Err(std::fmt::Error); - } - self.remaining -= 1; - - Ok(()) + fn write_timings_propagates_a_writer_failure_at_every_write() { + for timings in [ + timings_with_eval_counts(5, 6), + timings_with_eval_counts(0, 0), + ] { + let mut counting_writer = WriterCountingWrites { writes: 0 }; + assert_eq!(write_timings(&timings, &mut counting_writer), Ok(())); + + for remaining_writes in 0..counting_writer.writes { + assert_eq!( + write_timings(&timings, &mut WriterFailingAfterWrites { remaining_writes }), + Err(std::fmt::Error) + ); } } - - let with_counts = LlamaTimings::new(1.0, 2.0, 3.0, 4.0, 5, 6, 1); - let zero_counts = LlamaTimings::new(1.0, 2.0, 3.0, 4.0, 0, 0, 1); - - for writes_allowed in 0..20 { - let _ = super::write_timings( - &with_counts, - &mut FailAfterNWrites { - remaining: writes_allowed, - }, - ); - let _ = super::write_timings( - &zero_counts, - &mut FailAfterNWrites { - remaining: writes_allowed, - }, - ); - } } } diff --git a/llama-cpp-gbnf/src/gbnf_validation_error.rs b/llama-cpp-gbnf/src/gbnf_validation_error.rs index fe8891cd9..2d0097b52 100644 --- a/llama-cpp-gbnf/src/gbnf_validation_error.rs +++ b/llama-cpp-gbnf/src/gbnf_validation_error.rs @@ -16,8 +16,10 @@ pub enum GbnfValidationError { EmptyRuleSet, #[error("grammar does not define the root symbol {root:?}")] RootSymbolMissing { root: String }, - #[error("grammar is left-recursive and cannot be compiled by llama.cpp")] - LeftRecursion, + #[error( + "grammar is left-recursive or chains rules deeper than llama.cpp's recursion limit, so llama.cpp cannot compile it" + )] + UnsupportedGrammar, #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-gbnf/src/validate_gbnf.rs b/llama-cpp-gbnf/src/validate_gbnf.rs index 60f71e848..1e38c9c4a 100644 --- a/llama-cpp-gbnf/src/validate_gbnf.rs +++ b/llama-cpp-gbnf/src/validate_gbnf.rs @@ -6,11 +6,11 @@ use llama_cpp_ffi_status::FfiStatusError; use llama_cpp_ffi_status::read_and_free_cpp_string; use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_EMPTY_RULE_SET; -use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_LEFT_RECURSION; use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_OK; use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_ROOT_SYMBOL_MISSING; use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_SYNTAX_ERROR; use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_THREW_CXX_EXCEPTION; +use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_UNSUPPORTED_GRAMMAR; use llama_cpp_bindings_sys::llama_rs_gbnf_validation_status; use llama_cpp_bindings_sys::llama_rs_validate_gbnf; use llama_cpp_bindings_sys::llama_vocab; @@ -31,7 +31,9 @@ fn validation_status_to_result( root: root.to_owned(), }) } - LLAMA_RS_GBNF_VALIDATION_LEFT_RECURSION => Err(GbnfValidationError::LeftRecursion), + LLAMA_RS_GBNF_VALIDATION_UNSUPPORTED_GRAMMAR => { + Err(GbnfValidationError::UnsupportedGrammar) + } llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_NULL_GRAMMAR_ARG => { Err(FfiContractError { operation: "llama_rs_validate_gbnf", @@ -104,6 +106,7 @@ pub fn validate_gbnf( #[cfg(test)] mod tests { use std::ffi::CString; + use std::fmt::Write; use llama_cpp_ffi_status::FfiContractError; use llama_cpp_ffi_status::FfiStatusError; @@ -151,7 +154,21 @@ mod tests { fn left_recursive_grammar_is_rejected() { assert_eq!( validate_gbnf(None, r#"root ::= root "x""#, "root"), - Err(GbnfValidationError::LeftRecursion) + Err(GbnfValidationError::UnsupportedGrammar) + ); + } + + #[test] + fn grammar_chaining_rules_past_the_recursion_limit_is_rejected() { + let mut grammar = String::from("root ::= rule0\n"); + for rule_index in 0..300 { + writeln!(grammar, "rule{rule_index} ::= rule{}", rule_index + 1).unwrap(); + } + grammar.push_str("rule300 ::= \"x\"\n"); + + assert_eq!( + validate_gbnf(None, &grammar, "root"), + Err(GbnfValidationError::UnsupportedGrammar) ); } diff --git a/llama-cpp-wrapper-error-fixture/src/wrapper_allocated_error_message_pointer.rs b/llama-cpp-wrapper-error-fixture/src/wrapper_allocated_error_message_pointer.rs index 60e414ae4..3342fa8e9 100644 --- a/llama-cpp-wrapper-error-fixture/src/wrapper_allocated_error_message_pointer.rs +++ b/llama-cpp-wrapper-error-fixture/src/wrapper_allocated_error_message_pointer.rs @@ -20,7 +20,6 @@ pub fn wrapper_allocated_error_message_pointer() -> *mut c_char { let status = unsafe { llama_cpp_bindings_sys::llama_rs_json_schema_to_grammar( schema.as_ptr(), - false, &raw mut out_grammar, &raw mut out_error, ) @@ -28,7 +27,7 @@ pub fn wrapper_allocated_error_message_pointer() -> *mut c_char { assert_eq!( status, - llama_cpp_bindings_sys::LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_LLAMA_CPP_THREW_CXX_EXCEPTION, + llama_cpp_bindings_sys::LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_INVALID_JSON, "the fixture depends on the llama.cpp json parser rejecting this schema" ); assert!( diff --git a/llama-cpp-wrapper-sources/src/wrapper_sources.rs b/llama-cpp-wrapper-sources/src/wrapper_sources.rs index 9d231ce60..b9bd9ab98 100644 --- a/llama-cpp-wrapper-sources/src/wrapper_sources.rs +++ b/llama-cpp-wrapper-sources/src/wrapper_sources.rs @@ -1,4 +1,5 @@ pub const WRAPPER_SOURCES: &[&str] = &[ + "wrapper_batch.cpp", "wrapper_chat_apply.cpp", "wrapper_chat_parse.cpp", "wrapper_common.cpp", From 855a0002c367d0864f9cd62b038840ffa58152f6 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Fri, 30 Oct 2026 08:55:25 +0100 Subject: [PATCH 02/82] Render, constrain and parse chats through llama.cpp chat sessions and split LLM tests into intention-named modules --- .../src/bindgen_config.rs | 12 +- llama-cpp-bindings-sys/src/lib.rs | 6 - llama-cpp-bindings-sys/wrapper.h | 7 +- llama-cpp-bindings-sys/wrapper_batch.cpp | 17 +- llama-cpp-bindings-sys/wrapper_chat_apply.cpp | 95 - llama-cpp-bindings-sys/wrapper_chat_apply.h | 39 - llama-cpp-bindings-sys/wrapper_chat_parse.cpp | 446 --- llama-cpp-bindings-sys/wrapper_chat_parse.h | 199 -- .../wrapper_chat_session.cpp | 580 ++++ llama-cpp-bindings-sys/wrapper_chat_session.h | 229 ++ .../wrapper_chat_templates.cpp | 88 + .../wrapper_chat_templates.h | 45 + .../wrapper_chat_templates_handle.h | 8 + .../wrapper_parsed_chat.cpp | 61 + llama-cpp-bindings-sys/wrapper_parsed_chat.h | 50 + .../wrapper_parsed_chat_handle.h | 7 + llama-cpp-bindings-sys/wrapper_reasoning.cpp | 158 - llama-cpp-bindings-sys/wrapper_reasoning.h | 49 - llama-cpp-bindings-sys/wrapper_token_text.cpp | 20 - llama-cpp-bindings-sys/wrapper_token_text.h | 11 - llama-cpp-bindings-sys/wrapper_tool_calls.cpp | 236 -- llama-cpp-bindings-sys/wrapper_tool_calls.h | 48 - llama-cpp-bindings-sys/wrapper_utils.h | 5 + .../build_user_prompt_with_media_marker.rs | 16 - llama-cpp-bindings-tests/src/chat_answer.rs | 126 + .../src/chat_answer_generation.rs | 58 + .../src/classify_sample_loop.rs | 120 +- .../src/classify_sample_loop_outcome.rs | 109 + .../src/closed_reasoning_chat_session.rs | 43 + .../src/get_weather_tools_json.rs | 16 + .../src/ingested_generation.rs | 46 + llama-cpp-bindings-tests/src/lib.rs | 14 +- .../src/media_question_chat_session.rs | 24 + llama-cpp-bindings-tests/src/normalize.rs | 9 + .../src/paris_weather_tool_call.rs | 14 + .../src/penalized_nucleus_sampler.rs | 22 + .../src/prompt_question.rs | 1 + .../src/recognized_answer.rs | 25 + .../src/user_turn_chat_session.rs | 34 + .../approximate_tok_env_eos_selection.rs | 75 + .../tests/backend_initialization.rs | 5 +- .../tests/chat_answer_parsing.rs | 199 ++ .../tests/chat_prompt_rendering.rs | 109 + .../tests/chat_session_input_validation.rs | 58 + .../tests/chat_template_lookup.rs | 79 + .../tests/chat_template_tool_call_support.rs | 17 + .../tests/completion_usage_accounting.rs | 301 ++ .../tests/context_creation_limits.rs | 97 + .../tests/context_dimensions.rs | 87 + .../tests/context_state.rs | 2804 ----------------- .../tests/context_synchronization.rs | 24 + .../tests/debug_formatting.rs | 40 + .../tests/decode_abort_control.rs | 103 + .../tests/decode_batch_rejection.rs | 104 + .../tests/detokenization.rs | 326 ++ llama-cpp-bindings-tests/tests/dry_sampler.rs | 55 + .../tests/duck_typed_tool_calls.rs | 85 + .../tests/embedding_generation.rs | 334 ++ .../tests/embedding_models.rs | 830 ----- .../tests/embedding_retrieval_errors.rs | 119 + .../tests/encode_batch_rejection.rs | 41 + .../tests/encoder_requirement.rs | 63 + .../end_of_generation_token_detection.rs | 122 + .../tests/generation_control.rs | 2427 -------------- .../tests/generation_end_classification.rs | 109 + .../tests/grammar_sampler.rs | 293 ++ ...n_states.rs => hidden_state_extraction.rs} | 112 +- .../json_schema_constrained_generation.rs | 213 ++ .../tests/kv_cache_memory_availability.rs | 85 + .../tests/kv_cache_occupancy.rs | 133 + .../tests/kv_cache_position_bounds.rs | 388 +++ ...cache_position_shift_rope_compatibility.rs | 113 + .../tests/kv_cache_priming.rs | 45 + .../tests/kv_cache_sequence_edits.rs | 133 + .../tests/kv_cache_sequence_id_bounds.rs | 98 + .../tests/lazy_grammar_sampler.rs | 138 + .../tests/llguidance_sampler.rs | 599 ++++ .../tests/logit_bias_sampler.rs | 17 + .../tests/logits_retrieval.rs | 91 + .../tests/lora_adapters.rs | 63 + llama-cpp-bindings-tests/tests/main.rs | 85 +- .../tests/model_architecture.rs | 136 + .../tests/model_derived_data_caching.rs | 30 +- .../tests/model_hyperparameters.rs | 193 ++ .../tests/model_introspection.rs | 1855 ----------- .../tests/model_loading_errors.rs | 14 +- .../{chat_protocol.rs => model_metadata.rs} | 199 +- .../tests/multimodal_audio.rs | 30 +- .../tests/multimodal_bitmap_construction.rs | 67 + .../tests/multimodal_chunk_copy.rs | 47 + .../tests/multimodal_chunk_encoding.rs | 51 + .../tests/multimodal_chunk_evaluation.rs | 678 ++++ .../tests/multimodal_chunk_tokenization.rs | 317 ++ .../tests/multimodal_fusion.rs | 30 +- .../multimodal_projector_capabilities.rs | 141 + .../tests/multimodal_projector_loading.rs | 29 + .../multimodal_prompt_parts_tokenization.rs | 4 +- .../multimodal_reasoning_classification.rs | 418 +++ .../tests/multimodal_usage_accounting.rs | 301 ++ .../tests/multimodal_vision.rs | 2172 ------------- .../tests/multimodal_vision_inference.rs | 143 + .../tests/parameter_fitting.rs | 126 + .../tests/performance_timings.rs | 26 + .../tests/prompt_token_accounting.rs | 254 ++ .../tests/reasoning_classification.rs | 309 ++ llama-cpp-bindings-tests/tests/reranking.rs | 147 + .../tests/sampled_token_ingestion.rs | 205 ++ .../tests/sampler_token_acceptance.rs | 95 + .../tests/session_file_path_validation.rs | 363 +++ .../tests/session_file_token_capacity.rs | 231 ++ .../tests/session_files.rs | 295 ++ .../tests/special_tokens.rs | 197 ++ .../tests/state_snapshots.rs | 257 ++ .../tests/structured_chat_output.rs | 2357 -------------- .../tests/threadpool_detachment.rs | 24 + .../tests/token_attributes.rs | 48 + .../tests/token_candidates.rs | 165 + .../tests/tokenization.rs | 224 ++ .../tests/tokenize_synthetic.rs | 19 + .../tests/tool_call_classification.rs | 78 + .../tests/tool_call_parsing.rs | 319 ++ .../tests/unconstrained_token_sampling.rs | 214 ++ .../tests/vocabulary_bounds.rs | 61 + .../tests/vocabulary_enumeration.rs | 92 + .../tests/vocabulary_type.rs | 43 + llama-cpp-bindings-types/src/lib.rs | 2 - .../src/reasoning_markers.rs | 5 - .../src/bare_json_tool_call_probe.rs | 210 ++ .../src/chat_assistant_prefill.rs | 7 + llama-cpp-bindings/src/chat_assistant_turn.rs | 34 + llama-cpp-bindings/src/chat_classification.rs | 7 + .../src/chat_grammar_trigger.rs | 9 + .../src/chat_message_parse_outcome.rs | 49 - llama-cpp-bindings/src/chat_message_parser.rs | 2633 ---------------- llama-cpp-bindings/src/chat_output.rs | 108 + .../src/chat_output_completeness.rs | 5 + llama-cpp-bindings/src/chat_output_token.rs | 9 + llama-cpp-bindings/src/chat_piece_role.rs | 16 + llama-cpp-bindings/src/chat_prompt_part.rs | 14 + llama-cpp-bindings/src/chat_session.rs | 1538 +++++++++ .../src/chat_session_options.rs | 28 + llama-cpp-bindings/src/chat_span.rs | 9 + llama-cpp-bindings/src/chat_span_kind.rs | 116 + llama-cpp-bindings/src/chat_spans.rs | 248 ++ .../src/chat_template_tool_calls.rs | 3 - .../src/chat_template_tool_calls/detect.rs | 47 - .../gemma4_call_block.rs | 56 - .../glm47_key_value_tags.rs | 47 - .../known_marker_candidates.rs | 55 +- .../mistral3_arrow_args.rs | 50 - .../qwen3_json_inside_tool_call.rs | 64 - .../chat_template_tool_calls/qwen_xml_tags.rs | 53 - .../tool_call_format_registration.rs | 40 - llama-cpp-bindings/src/chat_templates.rs | 287 ++ llama-cpp-bindings/src/chat_tool_choice.rs | 16 + llama-cpp-bindings/src/context.rs | 4 - .../src/context/kv_cache_type.rs | 248 +- llama-cpp-bindings/src/context/params.rs | 14 +- llama-cpp-bindings/src/cpp_text_view.rs | 20 + llama-cpp-bindings/src/error.rs | 10 +- .../src/error/apply_chat_template_error.rs | 17 - .../src/error/chat_session_error.rs | 45 + .../src/error/chat_templates_error.rs | 15 + llama-cpp-bindings/src/error/ingest_error.rs | 10 + .../src/error/marker_detection_error.rs | 47 - .../src/error/parse_chat_message_error.rs | 34 +- llama-cpp-bindings/src/error/sample_error.rs | 6 +- ...extract_tool_call_markers_from_haystack.rs | 143 - llama-cpp-bindings/src/lib.rs | 66 +- llama-cpp-bindings/src/marker_role.rs | 79 - .../src/marker_role_candidate.rs | 12 - llama-cpp-bindings/src/model.rs | 2331 +------------- llama-cpp-bindings/src/model/params.rs | 28 +- llama-cpp-bindings/src/parsed_chat.rs | 237 ++ llama-cpp-bindings/src/raw_chat_message.rs | 22 +- .../src/resolved_tool_call_markers.rs | 5 - .../src/result_after_release.rs | 50 + .../src/sampled_token_classifier.rs | 1672 ++-------- llama-cpp-bindings/src/streaming_marker.rs | 91 - llama-cpp-bindings/src/streaming_markers.rs | 234 -- .../src/synthetic_tool_call_renders.rs | 5 - .../src/tool_call_marker_pair.rs | 5 - llama-cpp-test-harness-macros/src/expand.rs | 413 +-- llama-cpp-test-harness/src/execution_plan.rs | 11 +- llama-cpp-test-harness/src/llama_test_fn.rs | 28 +- llama-cpp-test-harness/src/no_op.rs | 7 +- llama-cpp-test-harness/src/scheduled_phase.rs | 21 +- .../tests/harness_self_test.rs | 25 +- .../src/wrapper_headers.rs | 11 +- .../src/wrapper_sources.rs | 8 +- .../tests/wrapper_files_on_disk.rs | 39 + 191 files changed, 16988 insertions(+), 22106 deletions(-) delete mode 100644 llama-cpp-bindings-sys/wrapper_chat_apply.cpp delete mode 100644 llama-cpp-bindings-sys/wrapper_chat_apply.h delete mode 100644 llama-cpp-bindings-sys/wrapper_chat_parse.cpp delete mode 100644 llama-cpp-bindings-sys/wrapper_chat_parse.h create mode 100644 llama-cpp-bindings-sys/wrapper_chat_session.cpp create mode 100644 llama-cpp-bindings-sys/wrapper_chat_session.h create mode 100644 llama-cpp-bindings-sys/wrapper_chat_templates.cpp create mode 100644 llama-cpp-bindings-sys/wrapper_chat_templates.h create mode 100644 llama-cpp-bindings-sys/wrapper_chat_templates_handle.h create mode 100644 llama-cpp-bindings-sys/wrapper_parsed_chat.cpp create mode 100644 llama-cpp-bindings-sys/wrapper_parsed_chat.h create mode 100644 llama-cpp-bindings-sys/wrapper_parsed_chat_handle.h delete mode 100644 llama-cpp-bindings-sys/wrapper_reasoning.cpp delete mode 100644 llama-cpp-bindings-sys/wrapper_reasoning.h delete mode 100644 llama-cpp-bindings-sys/wrapper_token_text.cpp delete mode 100644 llama-cpp-bindings-sys/wrapper_token_text.h delete mode 100644 llama-cpp-bindings-sys/wrapper_tool_calls.cpp delete mode 100644 llama-cpp-bindings-sys/wrapper_tool_calls.h delete mode 100644 llama-cpp-bindings-tests/src/build_user_prompt_with_media_marker.rs create mode 100644 llama-cpp-bindings-tests/src/chat_answer.rs create mode 100644 llama-cpp-bindings-tests/src/chat_answer_generation.rs create mode 100644 llama-cpp-bindings-tests/src/classify_sample_loop_outcome.rs create mode 100644 llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs create mode 100644 llama-cpp-bindings-tests/src/get_weather_tools_json.rs create mode 100644 llama-cpp-bindings-tests/src/ingested_generation.rs create mode 100644 llama-cpp-bindings-tests/src/media_question_chat_session.rs create mode 100644 llama-cpp-bindings-tests/src/normalize.rs create mode 100644 llama-cpp-bindings-tests/src/paris_weather_tool_call.rs create mode 100644 llama-cpp-bindings-tests/src/penalized_nucleus_sampler.rs create mode 100644 llama-cpp-bindings-tests/src/prompt_question.rs create mode 100644 llama-cpp-bindings-tests/src/recognized_answer.rs create mode 100644 llama-cpp-bindings-tests/src/user_turn_chat_session.rs create mode 100644 llama-cpp-bindings-tests/tests/approximate_tok_env_eos_selection.rs create mode 100644 llama-cpp-bindings-tests/tests/chat_answer_parsing.rs create mode 100644 llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs create mode 100644 llama-cpp-bindings-tests/tests/chat_session_input_validation.rs create mode 100644 llama-cpp-bindings-tests/tests/chat_template_lookup.rs create mode 100644 llama-cpp-bindings-tests/tests/chat_template_tool_call_support.rs create mode 100644 llama-cpp-bindings-tests/tests/completion_usage_accounting.rs create mode 100644 llama-cpp-bindings-tests/tests/context_creation_limits.rs create mode 100644 llama-cpp-bindings-tests/tests/context_dimensions.rs delete mode 100644 llama-cpp-bindings-tests/tests/context_state.rs create mode 100644 llama-cpp-bindings-tests/tests/context_synchronization.rs create mode 100644 llama-cpp-bindings-tests/tests/debug_formatting.rs create mode 100644 llama-cpp-bindings-tests/tests/decode_abort_control.rs create mode 100644 llama-cpp-bindings-tests/tests/decode_batch_rejection.rs create mode 100644 llama-cpp-bindings-tests/tests/detokenization.rs create mode 100644 llama-cpp-bindings-tests/tests/dry_sampler.rs create mode 100644 llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs create mode 100644 llama-cpp-bindings-tests/tests/embedding_generation.rs delete mode 100644 llama-cpp-bindings-tests/tests/embedding_models.rs create mode 100644 llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs create mode 100644 llama-cpp-bindings-tests/tests/encode_batch_rejection.rs create mode 100644 llama-cpp-bindings-tests/tests/encoder_requirement.rs create mode 100644 llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs delete mode 100644 llama-cpp-bindings-tests/tests/generation_control.rs create mode 100644 llama-cpp-bindings-tests/tests/generation_end_classification.rs create mode 100644 llama-cpp-bindings-tests/tests/grammar_sampler.rs rename llama-cpp-bindings-tests/tests/{hidden_states.rs => hidden_state_extraction.rs} (94%) create mode 100644 llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs create mode 100644 llama-cpp-bindings-tests/tests/kv_cache_memory_availability.rs create mode 100644 llama-cpp-bindings-tests/tests/kv_cache_occupancy.rs create mode 100644 llama-cpp-bindings-tests/tests/kv_cache_position_bounds.rs create mode 100644 llama-cpp-bindings-tests/tests/kv_cache_position_shift_rope_compatibility.rs create mode 100644 llama-cpp-bindings-tests/tests/kv_cache_priming.rs create mode 100644 llama-cpp-bindings-tests/tests/kv_cache_sequence_edits.rs create mode 100644 llama-cpp-bindings-tests/tests/kv_cache_sequence_id_bounds.rs create mode 100644 llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs create mode 100644 llama-cpp-bindings-tests/tests/llguidance_sampler.rs create mode 100644 llama-cpp-bindings-tests/tests/logit_bias_sampler.rs create mode 100644 llama-cpp-bindings-tests/tests/logits_retrieval.rs create mode 100644 llama-cpp-bindings-tests/tests/lora_adapters.rs create mode 100644 llama-cpp-bindings-tests/tests/model_architecture.rs create mode 100644 llama-cpp-bindings-tests/tests/model_hyperparameters.rs delete mode 100644 llama-cpp-bindings-tests/tests/model_introspection.rs rename llama-cpp-bindings-tests/tests/{chat_protocol.rs => model_metadata.rs} (55%) create mode 100644 llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs create mode 100644 llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs create mode 100644 llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs create mode 100644 llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs create mode 100644 llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs create mode 100644 llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs create mode 100644 llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs create mode 100644 llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs create mode 100644 llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs delete mode 100644 llama-cpp-bindings-tests/tests/multimodal_vision.rs create mode 100644 llama-cpp-bindings-tests/tests/multimodal_vision_inference.rs create mode 100644 llama-cpp-bindings-tests/tests/parameter_fitting.rs create mode 100644 llama-cpp-bindings-tests/tests/performance_timings.rs create mode 100644 llama-cpp-bindings-tests/tests/prompt_token_accounting.rs create mode 100644 llama-cpp-bindings-tests/tests/reasoning_classification.rs create mode 100644 llama-cpp-bindings-tests/tests/reranking.rs create mode 100644 llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs create mode 100644 llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs create mode 100644 llama-cpp-bindings-tests/tests/session_file_path_validation.rs create mode 100644 llama-cpp-bindings-tests/tests/session_file_token_capacity.rs create mode 100644 llama-cpp-bindings-tests/tests/session_files.rs create mode 100644 llama-cpp-bindings-tests/tests/special_tokens.rs create mode 100644 llama-cpp-bindings-tests/tests/state_snapshots.rs delete mode 100644 llama-cpp-bindings-tests/tests/structured_chat_output.rs create mode 100644 llama-cpp-bindings-tests/tests/threadpool_detachment.rs create mode 100644 llama-cpp-bindings-tests/tests/token_attributes.rs create mode 100644 llama-cpp-bindings-tests/tests/token_candidates.rs create mode 100644 llama-cpp-bindings-tests/tests/tokenization.rs create mode 100644 llama-cpp-bindings-tests/tests/tokenize_synthetic.rs create mode 100644 llama-cpp-bindings-tests/tests/tool_call_classification.rs create mode 100644 llama-cpp-bindings-tests/tests/tool_call_parsing.rs create mode 100644 llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs create mode 100644 llama-cpp-bindings-tests/tests/vocabulary_bounds.rs create mode 100644 llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs create mode 100644 llama-cpp-bindings-tests/tests/vocabulary_type.rs delete mode 100644 llama-cpp-bindings-types/src/reasoning_markers.rs create mode 100644 llama-cpp-bindings/src/bare_json_tool_call_probe.rs create mode 100644 llama-cpp-bindings/src/chat_assistant_prefill.rs create mode 100644 llama-cpp-bindings/src/chat_assistant_turn.rs create mode 100644 llama-cpp-bindings/src/chat_classification.rs create mode 100644 llama-cpp-bindings/src/chat_grammar_trigger.rs delete mode 100644 llama-cpp-bindings/src/chat_message_parser.rs create mode 100644 llama-cpp-bindings/src/chat_output.rs create mode 100644 llama-cpp-bindings/src/chat_output_completeness.rs create mode 100644 llama-cpp-bindings/src/chat_output_token.rs create mode 100644 llama-cpp-bindings/src/chat_piece_role.rs create mode 100644 llama-cpp-bindings/src/chat_prompt_part.rs create mode 100644 llama-cpp-bindings/src/chat_session.rs create mode 100644 llama-cpp-bindings/src/chat_session_options.rs create mode 100644 llama-cpp-bindings/src/chat_span.rs create mode 100644 llama-cpp-bindings/src/chat_span_kind.rs create mode 100644 llama-cpp-bindings/src/chat_spans.rs delete mode 100644 llama-cpp-bindings/src/chat_template_tool_calls/detect.rs delete mode 100644 llama-cpp-bindings/src/chat_template_tool_calls/tool_call_format_registration.rs create mode 100644 llama-cpp-bindings/src/chat_templates.rs create mode 100644 llama-cpp-bindings/src/chat_tool_choice.rs create mode 100644 llama-cpp-bindings/src/cpp_text_view.rs delete mode 100644 llama-cpp-bindings/src/error/apply_chat_template_error.rs create mode 100644 llama-cpp-bindings/src/error/chat_session_error.rs create mode 100644 llama-cpp-bindings/src/error/chat_templates_error.rs create mode 100644 llama-cpp-bindings/src/error/ingest_error.rs delete mode 100644 llama-cpp-bindings/src/error/marker_detection_error.rs delete mode 100644 llama-cpp-bindings/src/extract_tool_call_markers_from_haystack.rs delete mode 100644 llama-cpp-bindings/src/marker_role.rs delete mode 100644 llama-cpp-bindings/src/marker_role_candidate.rs create mode 100644 llama-cpp-bindings/src/parsed_chat.rs delete mode 100644 llama-cpp-bindings/src/resolved_tool_call_markers.rs create mode 100644 llama-cpp-bindings/src/result_after_release.rs delete mode 100644 llama-cpp-bindings/src/streaming_marker.rs delete mode 100644 llama-cpp-bindings/src/streaming_markers.rs delete mode 100644 llama-cpp-bindings/src/synthetic_tool_call_renders.rs delete mode 100644 llama-cpp-bindings/src/tool_call_marker_pair.rs create mode 100644 llama-cpp-wrapper-sources/tests/wrapper_files_on_disk.rs diff --git a/llama-cpp-bindings-build/src/bindgen_config.rs b/llama-cpp-bindings-build/src/bindgen_config.rs index ef14c0ab2..ca38fff00 100644 --- a/llama-cpp-bindings-build/src/bindgen_config.rs +++ b/llama-cpp-bindings-build/src/bindgen_config.rs @@ -60,6 +60,8 @@ const DEPRECATED_FUNCTIONS: &[&str] = &[ "mtmd_image_tokens_get_ny", ]; +const COMPARED_STRUCTS: &[&str] = &["llama_logit_bias", "llama_token_data"]; + struct PrivatizedField { type_name: &'static str, field_name: &'static str, @@ -123,6 +125,14 @@ impl bindgen::callbacks::ParseCallbacks for BindingCallbacks { println!("cargo:rerun-if-env-changed={key}"); } + fn add_derives(&self, info: &bindgen::callbacks::DeriveInfo<'_>) -> Vec { + if COMPARED_STRUCTS.contains(&info.name) { + vec!["PartialEq".to_owned()] + } else { + Vec::new() + } + } + fn field_visibility( &self, info: bindgen::callbacks::FieldInfo<'_>, @@ -177,7 +187,7 @@ fn create_base_builder(llama_src: &Path, callbacks: BindingCallbacks) -> bindgen .clang_arg(format!("-I{}", llama_src.join("include").display())) .clang_arg(format!("-I{}", llama_src.join("ggml/include").display())) .parse_callbacks(Box::new(callbacks)) - .derive_partialeq(true) + .generate_comments(false) .allowlist_function("ggml_.*") .allowlist_type("ggml_.*") .allowlist_var("GGML_MAX_DIMS") diff --git a/llama-cpp-bindings-sys/src/lib.rs b/llama-cpp-bindings-sys/src/lib.rs index 0b6b4ba15..4b3b5f2a6 100644 --- a/llama-cpp-bindings-sys/src/lib.rs +++ b/llama-cpp-bindings-sys/src/lib.rs @@ -7,12 +7,6 @@ reason = "bindgen emits C struct and enum names verbatim and they don't follow Rust naming" )] #![expect( - unpredictable_function_pointer_comparisons, - reason = "bindgen-generated FFI function pointers are opaque and the lint cannot reason about them" -)] -#![expect( - clippy::derive_partial_eq_without_eq, - clippy::doc_markdown, clippy::pub_underscore_fields, clippy::use_self, reason = "bindgen writes this file from the llama.cpp headers; its shape is not ours to change" diff --git a/llama-cpp-bindings-sys/wrapper.h b/llama-cpp-bindings-sys/wrapper.h index d1062c26e..c7c0853d8 100644 --- a/llama-cpp-bindings-sys/wrapper.h +++ b/llama-cpp-bindings-sys/wrapper.h @@ -1,14 +1,13 @@ #include "llama.cpp/include/llama.h" #include "llama.cpp/ggml/include/gguf.h" #include "wrapper_batch.h" -#include "wrapper_chat_apply.h" -#include "wrapper_chat_parse.h" +#include "wrapper_chat_session.h" +#include "wrapper_chat_templates.h" #include "wrapper_common.h" #include "wrapper_context.h" #include "wrapper_fit.h" #include "wrapper_gbnf.h" #include "wrapper_mtmd.h" -#include "wrapper_reasoning.h" +#include "wrapper_parsed_chat.h" #include "wrapper_sampler_init.h" #include "wrapper_state.h" -#include "wrapper_tool_calls.h" diff --git a/llama-cpp-bindings-sys/wrapper_batch.cpp b/llama-cpp-bindings-sys/wrapper_batch.cpp index f1de0153c..0d2465ae5 100644 --- a/llama-cpp-bindings-sys/wrapper_batch.cpp +++ b/llama-cpp-bindings-sys/wrapper_batch.cpp @@ -1,6 +1,11 @@ #include "wrapper_batch.h" #include "wrapper_report_current_exception.h" +#include "llama.cpp/include/llama.h" + +#include +#include +#include #include extern "C" auto llama_rs_batch_ext_init( @@ -86,10 +91,14 @@ extern "C" auto llama_rs_batch_ext_add_entry( } } - for (const llama_seq_id seq_id : seq_id_span.subspan(1)) { - if (!llama_batch_ext_add_seq(batch, index, seq_id)) { - return LLAMA_RS_BATCH_EXT_ADD_ENTRY_INVALID_SEQUENCE_ID; - } + const gsl::span additional_seq_ids = seq_id_span.subspan(1); + const bool every_seq_id_added = std::all_of( + additional_seq_ids.begin(), additional_seq_ids.end(), [&](const llama_seq_id seq_id) -> bool { + return llama_batch_ext_add_seq(batch, index, seq_id); + }); + + if (!every_seq_id_added) { + return LLAMA_RS_BATCH_EXT_ADD_ENTRY_INVALID_SEQUENCE_ID; } if (!llama_batch_ext_set_pos(batch, index, &pos)) { diff --git a/llama-cpp-bindings-sys/wrapper_chat_apply.cpp b/llama-cpp-bindings-sys/wrapper_chat_apply.cpp deleted file mode 100644 index 505e711f6..000000000 --- a/llama-cpp-bindings-sys/wrapper_chat_apply.cpp +++ /dev/null @@ -1,95 +0,0 @@ -#include "wrapper_chat_apply.h" -#include "wrapper_token_text.h" - -#include "llama.cpp/common/chat-auto-parser.h" -#include "llama.cpp/common/chat.h" -#include "llama.cpp/common/json.h" -#include "llama.cpp/include/llama.h" -#include "wrapper_report_current_exception.h" -#include "wrapper_utils.h" - -#include -#include -#include -#include - -using wrapper_helpers::token_text_or_empty; - -extern "C" auto llama_rs_apply_chat_template( - const struct llama_model * model, - const char * template_src, - const char * const * roles, - const char * const * contents, - size_t n_messages, - int add_generation_prompt, - int enable_thinking, - char ** out_string, - char ** out_error) -> llama_rs_apply_chat_template_status { - if (out_string != nullptr) { - *out_string = nullptr; - } - if (out_error != nullptr) { - *out_error = nullptr; - } - if (model == nullptr) { - return LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_MODEL_ARG; - } - if (template_src == nullptr) { - return LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_TEMPLATE_ARG; - } - if (n_messages > 0 && ((roles == nullptr) || (contents == nullptr))) { - return LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_MESSAGES_ARG; - } - if (out_string == nullptr) { - return LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_OUT_STRING_ARG; - } - if (out_error == nullptr) { - return LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_OUT_ERROR_ARG; - } - - try { - const llama_vocab * vocab = llama_model_get_vocab(model); - if (vocab == nullptr) { - return LLAMA_RS_APPLY_CHAT_TEMPLATE_MODEL_HAS_NO_VOCAB; - } - - std::string const bos_token = token_text_or_empty(vocab, llama_vocab_bos(vocab)); - std::string const eos_token = token_text_or_empty(vocab, llama_vocab_eos(vocab)); - - common_chat_template const tmpl(template_src, bos_token, eos_token); - - common_json messages = common_json::array(); - const gsl::span role_span(roles, n_messages); - const gsl::span content_span(contents, n_messages); - for (size_t index = 0; index < n_messages; index++) { - messages.push_back({ - { "role", (role_span[index] != nullptr) ? role_span[index] : "" }, - { "content", (content_span[index] != nullptr) ? content_span[index] : "" }, - }); - } - - autoparser::generation_params inputs; - inputs.messages = std::move(messages); - inputs.tools = common_json::array(); - inputs.add_generation_prompt = add_generation_prompt != 0; - inputs.enable_thinking = enable_thinking != 0; - - std::string const rendered = common_chat_template_direct_apply(tmpl, inputs).str(); - if (rendered.empty()) { - return LLAMA_RS_APPLY_CHAT_TEMPLATE_TEMPLATE_APPLICATION_FAILED; - } - - *out_string = llama_rs_dup_string(rendered); - if (*out_string == nullptr) { - return LLAMA_RS_APPLY_CHAT_TEMPLATE_ERROR_STRING_ALLOCATION_FAILED; - } - - return LLAMA_RS_APPLY_CHAT_TEMPLATE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_APPLY_CHAT_TEMPLATE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_APPLY_CHAT_TEMPLATE_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_APPLY_CHAT_TEMPLATE_ERROR_STRING_ALLOCATION_FAILED); - } -} diff --git a/llama-cpp-bindings-sys/wrapper_chat_apply.h b/llama-cpp-bindings-sys/wrapper_chat_apply.h deleted file mode 100644 index 3add989a3..000000000 --- a/llama-cpp-bindings-sys/wrapper_chat_apply.h +++ /dev/null @@ -1,39 +0,0 @@ -#pragma once - -#include "llama.cpp/include/llama.h" -#include "wrapper_utils.h" - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -typedef enum llama_rs_apply_chat_template_status { - LLAMA_RS_APPLY_CHAT_TEMPLATE_OK = 0, - LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_MODEL_ARG, - LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_TEMPLATE_ARG, - LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_MESSAGES_ARG, - LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_OUT_STRING_ARG, - LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_OUT_ERROR_ARG, - LLAMA_RS_APPLY_CHAT_TEMPLATE_MODEL_HAS_NO_VOCAB, - LLAMA_RS_APPLY_CHAT_TEMPLATE_TEMPLATE_APPLICATION_FAILED, - LLAMA_RS_APPLY_CHAT_TEMPLATE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_APPLY_CHAT_TEMPLATE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_APPLY_CHAT_TEMPLATE_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_apply_chat_template_status; - -llama_rs_apply_chat_template_status llama_rs_apply_chat_template( - const struct llama_model * model, - const char * template_src, - const char * const * roles, - const char * const * contents, - size_t n_messages, - int add_generation_prompt, - int enable_thinking, - char ** out_string, - char ** out_error); - -#ifdef __cplusplus -} -#endif diff --git a/llama-cpp-bindings-sys/wrapper_chat_parse.cpp b/llama-cpp-bindings-sys/wrapper_chat_parse.cpp deleted file mode 100644 index fe661ac2d..000000000 --- a/llama-cpp-bindings-sys/wrapper_chat_parse.cpp +++ /dev/null @@ -1,446 +0,0 @@ -#include "wrapper_chat_parse.h" -#include "peg-parser.h" -#include "wrapper_token_text.h" - -#include "llama.cpp/common/chat-auto-parser.h" -#include "llama.cpp/common/chat.h" -#include "llama.cpp/common/json.h" -#include "llama.cpp/include/llama.h" -#include "wrapper_report_current_exception.h" -#include "wrapper_utils.h" - -#include -#include -#include -#include - -using wrapper_helpers::token_text_or_empty; - -struct llama_rs_parsed_chat { - common_chat_msg message; -}; - -struct llama_rs_chat_parser { - autoparser::autoparser parser; -}; - -struct llama_rs_chat_tools_parser { - common_peg_arena arena; -}; - -namespace { -void dup_or_set_alloc_flag(const std::string & source, char ** out_dup, bool * out_alloc_failed) { - *out_dup = llama_rs_dup_string(source); - *out_alloc_failed = (*out_dup == nullptr); -} - -auto parse_with_tools_parser( - const common_peg_arena & chat_parser, - const char * input, - int is_partial, - llama_rs_parsed_chat_handle * out_handle, - char ** out_error) -> llama_rs_parse_chat_message_status { - try { - common_chat_parser_params parser_params; - parser_params.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - - common_chat_msg parsed = - common_chat_peg_parse(chat_parser, common_chat_input(std::string(input)), is_partial != 0, parser_params); - - auto handle = std::make_unique(); - handle->message = std::move(parsed); - - *out_handle = handle.release(); - - return LLAMA_RS_PARSE_CHAT_MESSAGE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_PARSE_CHAT_MESSAGE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSE_CHAT_MESSAGE_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_PARSE_CHAT_MESSAGE_ERROR_STRING_ALLOCATION_FAILED); - } -} -} // namespace - -extern "C" auto llama_rs_chat_parser_create( - const struct llama_model * model, - llama_rs_chat_parser_handle * out_parser, - char ** out_error) -> llama_rs_chat_parser_create_status { - if (out_parser != nullptr) { - *out_parser = nullptr; - } - if (out_error != nullptr) { - *out_error = nullptr; - } - if (model == nullptr) { - return LLAMA_RS_CHAT_PARSER_CREATE_NULL_MODEL_ARG; - } - if (out_parser == nullptr) { - return LLAMA_RS_CHAT_PARSER_CREATE_NULL_OUT_PARSER_ARG; - } - if (out_error == nullptr) { - return LLAMA_RS_CHAT_PARSER_CREATE_NULL_OUT_ERROR_ARG; - } - - try { - const char * tmpl_src = llama_model_chat_template(model, nullptr); - if (tmpl_src == nullptr) { - return LLAMA_RS_CHAT_PARSER_CREATE_MODEL_HAS_NO_CHAT_TEMPLATE; - } - - const llama_vocab * vocab = llama_model_get_vocab(model); - if (vocab == nullptr) { - return LLAMA_RS_CHAT_PARSER_CREATE_MODEL_HAS_NO_VOCAB; - } - - std::string const bos_token = token_text_or_empty(vocab, llama_vocab_bos(vocab)); - std::string const eos_token = token_text_or_empty(vocab, llama_vocab_eos(vocab)); - - common_chat_template const tmpl(tmpl_src, bos_token, eos_token); - - auto parser_handle = std::make_unique(); - parser_handle->parser.analyze_template(tmpl); - - *out_parser = parser_handle.release(); - - return LLAMA_RS_CHAT_PARSER_CREATE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_CHAT_PARSER_CREATE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_PARSER_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_CHAT_PARSER_CREATE_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_chat_parser_free( - llama_rs_chat_parser_handle parser, - char ** out_error) -> llama_rs_chat_parser_free_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - try { - const std::unique_ptr reclaimed(parser); - return LLAMA_RS_CHAT_PARSER_FREE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_CHAT_PARSER_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_PARSER_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - LLAMA_RS_CHAT_PARSER_FREE_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_chat_tools_parser_create( - llama_rs_chat_parser_handle parser, - const char * tools_json, - llama_rs_chat_tools_parser_handle * out_tools_parser, - char ** out_error) -> llama_rs_chat_tools_parser_create_status { - if (out_tools_parser != nullptr) { - *out_tools_parser = nullptr; - } - if (out_error != nullptr) { - *out_error = nullptr; - } - if (parser == nullptr) { - return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_PARSER_ARG; - } - if (tools_json == nullptr) { - return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_TOOLS_JSON_ARG; - } - if (out_tools_parser == nullptr) { - return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_OUT_TOOLS_PARSER_ARG; - } - if (out_error == nullptr) { - return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_OUT_ERROR_ARG; - } - - try { - autoparser::generation_params inputs; - - inputs.tools = common_json::parse(tools_json); - - if (!inputs.tools.is_array()) { - return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_TOOLS_NOT_AN_ARRAY; - } - - auto tools_parser_handle = std::make_unique(); - tools_parser_handle->arena = parser->parser.build_parser(inputs, std::string()); - - *out_tools_parser = tools_parser_handle.release(); - - return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_chat_tools_parser_free( - llama_rs_chat_tools_parser_handle tools_parser, - char ** out_error) -> llama_rs_chat_tools_parser_free_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - try { - const std::unique_ptr reclaimed(tools_parser); - return LLAMA_RS_CHAT_TOOLS_PARSER_FREE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_CHAT_TOOLS_PARSER_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_TOOLS_PARSER_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - LLAMA_RS_CHAT_TOOLS_PARSER_FREE_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_parse_chat_message( - llama_rs_chat_tools_parser_handle tools_parser, - const char * input, - int is_partial, - llama_rs_parsed_chat_handle * out_handle, - char ** out_error) -> llama_rs_parse_chat_message_status { - if (out_handle != nullptr) { - *out_handle = nullptr; - } - if (out_error != nullptr) { - *out_error = nullptr; - } - if (tools_parser == nullptr) { - return LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_TOOLS_PARSER_ARG; - } - if (input == nullptr) { - return LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_INPUT_ARG; - } - if (out_handle == nullptr) { - return LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_OUT_HANDLE_ARG; - } - if (out_error == nullptr) { - return LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_OUT_ERROR_ARG; - } - - return parse_with_tools_parser(tools_parser->arena, input, is_partial, out_handle, out_error); -} - -extern "C" auto llama_rs_parsed_chat_free( - llama_rs_parsed_chat_handle handle, - char ** out_error) -> llama_rs_parsed_chat_free_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - try { - const std::unique_ptr reclaimed(handle); - return LLAMA_RS_PARSED_CHAT_FREE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_PARSED_CHAT_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - LLAMA_RS_PARSED_CHAT_FREE_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_parsed_chat_tool_call_count( - llama_rs_parsed_chat_handle handle, - size_t * out_count, - char ** out_error) -> llama_rs_parsed_chat_tool_call_count_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - if (out_count != nullptr) { - *out_count = 0; - } - if (handle == nullptr) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_NULL_HANDLE_ARG; - } - if (out_count == nullptr) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_NULL_OUT_COUNT_ARG; - } - try { - *out_count = handle->message.tool_calls.size(); - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_parsed_chat_tool_call_id( - llama_rs_parsed_chat_handle handle, - size_t index, - char ** out_string, - char ** out_error) -> llama_rs_parsed_chat_tool_call_id_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - if (out_string != nullptr) { - *out_string = nullptr; - } - if (handle == nullptr) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_NULL_HANDLE_ARG; - } - if (out_string == nullptr) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_NULL_OUT_STRING_ARG; - } - try { - if (index >= handle->message.tool_calls.size()) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_INDEX_OUT_OF_BOUNDS; - } - bool alloc_failed = false; - dup_or_set_alloc_flag(handle->message.tool_calls[index].id, out_string, &alloc_failed); - if (alloc_failed) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_ERROR_STRING_ALLOCATION_FAILED; - } - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_parsed_chat_tool_call_name( - llama_rs_parsed_chat_handle handle, - size_t index, - char ** out_string, - char ** out_error) -> llama_rs_parsed_chat_tool_call_name_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - if (out_string != nullptr) { - *out_string = nullptr; - } - if (handle == nullptr) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_NULL_HANDLE_ARG; - } - if (out_string == nullptr) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_NULL_OUT_STRING_ARG; - } - try { - if (index >= handle->message.tool_calls.size()) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_INDEX_OUT_OF_BOUNDS; - } - bool alloc_failed = false; - dup_or_set_alloc_flag(handle->message.tool_calls[index].name, out_string, &alloc_failed); - if (alloc_failed) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_ERROR_STRING_ALLOCATION_FAILED; - } - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_parsed_chat_tool_call_arguments( - llama_rs_parsed_chat_handle handle, - size_t index, - char ** out_string, - char ** out_error) -> llama_rs_parsed_chat_tool_call_arguments_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - if (out_string != nullptr) { - *out_string = nullptr; - } - if (handle == nullptr) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NULL_HANDLE_ARG; - } - if (out_string == nullptr) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NULL_OUT_STRING_ARG; - } - try { - if (index >= handle->message.tool_calls.size()) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_INDEX_OUT_OF_BOUNDS; - } - bool alloc_failed = false; - dup_or_set_alloc_flag( - handle->message.tool_calls[index].arguments, out_string, &alloc_failed); - if (alloc_failed) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_ERROR_STRING_ALLOCATION_FAILED; - } - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_parsed_chat_content( - llama_rs_parsed_chat_handle handle, - char ** out_string, - char ** out_error) -> llama_rs_parsed_chat_content_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - if (out_string != nullptr) { - *out_string = nullptr; - } - if (handle == nullptr) { - return LLAMA_RS_PARSED_CHAT_CONTENT_NULL_HANDLE_ARG; - } - if (out_string == nullptr) { - return LLAMA_RS_PARSED_CHAT_CONTENT_NULL_OUT_STRING_ARG; - } - try { - bool alloc_failed = false; - dup_or_set_alloc_flag(handle->message.content, out_string, &alloc_failed); - if (alloc_failed) { - return LLAMA_RS_PARSED_CHAT_CONTENT_ERROR_STRING_ALLOCATION_FAILED; - } - return LLAMA_RS_PARSED_CHAT_CONTENT_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_PARSED_CHAT_CONTENT_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_CONTENT_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_PARSED_CHAT_CONTENT_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_parsed_chat_reasoning_content( - llama_rs_parsed_chat_handle handle, - char ** out_string, - char ** out_error) -> llama_rs_parsed_chat_reasoning_content_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - if (out_string != nullptr) { - *out_string = nullptr; - } - if (handle == nullptr) { - return LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_NULL_HANDLE_ARG; - } - if (out_string == nullptr) { - return LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_NULL_OUT_STRING_ARG; - } - try { - bool alloc_failed = false; - dup_or_set_alloc_flag(handle->message.reasoning_content, out_string, &alloc_failed); - if (alloc_failed) { - return LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_ERROR_STRING_ALLOCATION_FAILED; - } - return LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_ERROR_STRING_ALLOCATION_FAILED); - } -} diff --git a/llama-cpp-bindings-sys/wrapper_chat_parse.h b/llama-cpp-bindings-sys/wrapper_chat_parse.h deleted file mode 100644 index 181d1efb7..000000000 --- a/llama-cpp-bindings-sys/wrapper_chat_parse.h +++ /dev/null @@ -1,199 +0,0 @@ -#pragma once - -#include "llama.cpp/include/llama.h" -#include "wrapper_utils.h" - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -struct llama_rs_parsed_chat; -typedef struct llama_rs_parsed_chat * llama_rs_parsed_chat_handle; - -struct llama_rs_chat_parser; -typedef struct llama_rs_chat_parser * llama_rs_chat_parser_handle; - -struct llama_rs_chat_tools_parser; -typedef struct llama_rs_chat_tools_parser * llama_rs_chat_tools_parser_handle; - -typedef enum llama_rs_chat_parser_create_status { - LLAMA_RS_CHAT_PARSER_CREATE_OK = 0, - LLAMA_RS_CHAT_PARSER_CREATE_NULL_MODEL_ARG, - LLAMA_RS_CHAT_PARSER_CREATE_NULL_OUT_PARSER_ARG, - LLAMA_RS_CHAT_PARSER_CREATE_NULL_OUT_ERROR_ARG, - LLAMA_RS_CHAT_PARSER_CREATE_MODEL_HAS_NO_CHAT_TEMPLATE, - LLAMA_RS_CHAT_PARSER_CREATE_MODEL_HAS_NO_VOCAB, - LLAMA_RS_CHAT_PARSER_CREATE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_CHAT_PARSER_CREATE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_PARSER_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_chat_parser_create_status; - -llama_rs_chat_parser_create_status llama_rs_chat_parser_create( - const struct llama_model * model, - llama_rs_chat_parser_handle * out_parser, - char ** out_error); - -typedef enum llama_rs_chat_parser_free_status { - LLAMA_RS_CHAT_PARSER_FREE_OK = 0, - LLAMA_RS_CHAT_PARSER_FREE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_CHAT_PARSER_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_PARSER_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, -} llama_rs_chat_parser_free_status; - -llama_rs_chat_parser_free_status llama_rs_chat_parser_free( - llama_rs_chat_parser_handle parser, - char ** out_error); - -typedef enum llama_rs_chat_tools_parser_create_status { - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_OK = 0, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_PARSER_ARG, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_TOOLS_JSON_ARG, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_OUT_TOOLS_PARSER_ARG, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_OUT_ERROR_ARG, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_TOOLS_NOT_AN_ARRAY, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_chat_tools_parser_create_status; - -llama_rs_chat_tools_parser_create_status llama_rs_chat_tools_parser_create( - llama_rs_chat_parser_handle parser, - const char * tools_json, - llama_rs_chat_tools_parser_handle * out_tools_parser, - char ** out_error); - -typedef enum llama_rs_chat_tools_parser_free_status { - LLAMA_RS_CHAT_TOOLS_PARSER_FREE_OK = 0, - LLAMA_RS_CHAT_TOOLS_PARSER_FREE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_CHAT_TOOLS_PARSER_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_TOOLS_PARSER_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, -} llama_rs_chat_tools_parser_free_status; - -llama_rs_chat_tools_parser_free_status llama_rs_chat_tools_parser_free( - llama_rs_chat_tools_parser_handle tools_parser, - char ** out_error); - -typedef enum llama_rs_parse_chat_message_status { - LLAMA_RS_PARSE_CHAT_MESSAGE_OK = 0, - LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_TOOLS_PARSER_ARG, - LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_INPUT_ARG, - LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_OUT_HANDLE_ARG, - LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_OUT_ERROR_ARG, - LLAMA_RS_PARSE_CHAT_MESSAGE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_PARSE_CHAT_MESSAGE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSE_CHAT_MESSAGE_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_parse_chat_message_status; - -llama_rs_parse_chat_message_status llama_rs_parse_chat_message( - llama_rs_chat_tools_parser_handle tools_parser, - const char * input, - int is_partial, - llama_rs_parsed_chat_handle * out_handle, - char ** out_error); - -typedef enum llama_rs_parsed_chat_free_status { - LLAMA_RS_PARSED_CHAT_FREE_OK = 0, - LLAMA_RS_PARSED_CHAT_FREE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_PARSED_CHAT_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, -} llama_rs_parsed_chat_free_status; - -llama_rs_parsed_chat_free_status llama_rs_parsed_chat_free( - llama_rs_parsed_chat_handle handle, - char ** out_error); - -typedef enum llama_rs_parsed_chat_tool_call_count_status { - LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_OK = 0, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_NULL_HANDLE_ARG, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_NULL_OUT_COUNT_ARG, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_parsed_chat_tool_call_count_status; - -llama_rs_parsed_chat_tool_call_count_status llama_rs_parsed_chat_tool_call_count( - llama_rs_parsed_chat_handle handle, - size_t * out_count, - char ** out_error); - -typedef enum llama_rs_parsed_chat_tool_call_id_status { - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_OK = 0, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_NULL_HANDLE_ARG, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_NULL_OUT_STRING_ARG, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_INDEX_OUT_OF_BOUNDS, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_parsed_chat_tool_call_id_status; - -llama_rs_parsed_chat_tool_call_id_status llama_rs_parsed_chat_tool_call_id( - llama_rs_parsed_chat_handle handle, - size_t index, - char ** out_string, - char ** out_error); - -typedef enum llama_rs_parsed_chat_tool_call_name_status { - LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_OK = 0, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_NULL_HANDLE_ARG, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_NULL_OUT_STRING_ARG, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_INDEX_OUT_OF_BOUNDS, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_parsed_chat_tool_call_name_status; - -llama_rs_parsed_chat_tool_call_name_status llama_rs_parsed_chat_tool_call_name( - llama_rs_parsed_chat_handle handle, - size_t index, - char ** out_string, - char ** out_error); - -typedef enum llama_rs_parsed_chat_tool_call_arguments_status { - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_OK = 0, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NULL_HANDLE_ARG, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NULL_OUT_STRING_ARG, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_INDEX_OUT_OF_BOUNDS, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_parsed_chat_tool_call_arguments_status; - -llama_rs_parsed_chat_tool_call_arguments_status llama_rs_parsed_chat_tool_call_arguments( - llama_rs_parsed_chat_handle handle, - size_t index, - char ** out_string, - char ** out_error); - -typedef enum llama_rs_parsed_chat_content_status { - LLAMA_RS_PARSED_CHAT_CONTENT_OK = 0, - LLAMA_RS_PARSED_CHAT_CONTENT_NULL_HANDLE_ARG, - LLAMA_RS_PARSED_CHAT_CONTENT_NULL_OUT_STRING_ARG, - LLAMA_RS_PARSED_CHAT_CONTENT_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_PARSED_CHAT_CONTENT_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_CONTENT_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_parsed_chat_content_status; - -llama_rs_parsed_chat_content_status llama_rs_parsed_chat_content( - llama_rs_parsed_chat_handle handle, - char ** out_string, - char ** out_error); - -typedef enum llama_rs_parsed_chat_reasoning_content_status { - LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_OK = 0, - LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_NULL_HANDLE_ARG, - LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_NULL_OUT_STRING_ARG, - LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_parsed_chat_reasoning_content_status; - -llama_rs_parsed_chat_reasoning_content_status llama_rs_parsed_chat_reasoning_content( - llama_rs_parsed_chat_handle handle, - char ** out_string, - char ** out_error); - -#ifdef __cplusplus -} -#endif diff --git a/llama-cpp-bindings-sys/wrapper_chat_session.cpp b/llama-cpp-bindings-sys/wrapper_chat_session.cpp new file mode 100644 index 000000000..2430db17a --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_chat_session.cpp @@ -0,0 +1,580 @@ +#include "wrapper_chat_session.h" +#include "wrapper_chat_templates.h" +#include "wrapper_chat_templates_handle.h" +#include "wrapper_parsed_chat.h" +#include "wrapper_parsed_chat_handle.h" +#include "wrapper_report_current_exception.h" +#include "wrapper_utils.h" + +#include "llama.cpp/common/chat-peg-parser.h" +#include "llama.cpp/common/chat.h" +#include "llama.cpp/common/common.h" +#include "llama.cpp/common/json.h" +#include "llama.cpp/common/peg-parser.h" +#include "llama.cpp/include/llama.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct llama_rs_chat_session { + common_chat_session session; + common_chat_input generation_prompt; + std::vector generation_prompt_tokens; + common_params_sampling sampling; + std::vector preserved_tokens; +}; + +struct llama_rs_chat_span { + llama_rs_chat_span_kind kind; + size_t start; + size_t end; +}; + +struct llama_rs_chat_spans { + size_t parsed_length = 0; + std::vector spans; +}; + +namespace { +auto tool_choice_of(llama_rs_chat_tool_choice tool_choice) -> common_chat_tool_choice { + switch (tool_choice) { + case LLAMA_RS_CHAT_TOOL_CHOICE_NONE: + return COMMON_CHAT_TOOL_CHOICE_NONE; + case LLAMA_RS_CHAT_TOOL_CHOICE_REQUIRED: + return COMMON_CHAT_TOOL_CHOICE_REQUIRED; + case LLAMA_RS_CHAT_TOOL_CHOICE_AUTO: + return COMMON_CHAT_TOOL_CHOICE_AUTO; + } + + return COMMON_CHAT_TOOL_CHOICE_AUTO; +} + +auto continuation_of(llama_rs_chat_assistant_turn assistant_turn) -> common_chat_continuation { + switch (assistant_turn) { + case LLAMA_RS_CHAT_ASSISTANT_TURN_CONTINUE_REASONING: + return COMMON_CHAT_CONTINUATION_REASONING; + case LLAMA_RS_CHAT_ASSISTANT_TURN_CONTINUE_CONTENT: + return COMMON_CHAT_CONTINUATION_CONTENT; + case LLAMA_RS_CHAT_ASSISTANT_TURN_OMITTED: + case LLAMA_RS_CHAT_ASSISTANT_TURN_NEW: + return COMMON_CHAT_CONTINUATION_NONE; + } + + return COMMON_CHAT_CONTINUATION_NONE; +} + +auto span_kind_of_tag(const std::string & tag) -> std::optional { + if (tag == common_chat_peg_builder::REASONING_BLOCK) { + return LLAMA_RS_CHAT_SPAN_KIND_REASONING_BLOCK; + } + if (tag == common_chat_peg_builder::REASONING) { + return LLAMA_RS_CHAT_SPAN_KIND_REASONING; + } + if (tag == common_chat_peg_builder::CONTENT) { + return LLAMA_RS_CHAT_SPAN_KIND_CONTENT; + } + if (tag == common_chat_peg_builder::TOOL_OPEN || tag == common_chat_peg_builder::TOOL_CLOSE || + tag == common_chat_peg_builder::TOOL_ARG_OPEN || tag == common_chat_peg_builder::TOOL_ARG_CLOSE) { + return LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL_MARKER; + } + if (tag == common_chat_peg_builder::TOOL || tag == common_chat_peg_builder::TOOL_ID || + tag == common_chat_peg_builder::TOOL_NAME || tag == common_chat_peg_builder::TOOL_ARGS || + tag == common_chat_peg_builder::TOOL_ARG || tag == common_chat_peg_builder::TOOL_ARG_NAME || + tag == common_chat_peg_builder::TOOL_ARG_VALUE || tag == common_chat_peg_builder::TOOL_ARG_STRING_VALUE || + tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_OBJECT || tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_ARRAY || + tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_ITEM) { + return LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL; + } + + return std::nullopt; +} + +auto is_trigger_rule(const common_peg_arena & parser, const std::string & rule) -> bool { + if (rule.empty() || !parser.has_rule(rule)) { + return false; + } + + const auto * rule_parser = std::get_if(&parser.get(parser.get_rule(rule))); + + return rule_parser != nullptr && rule_parser->trigger; +} + +auto span_kind_of_node(const common_peg_arena & parser, const common_peg_ast_node & node) + -> std::optional { + if (const auto kind = span_kind_of_tag(node.tag)) { + return kind; + } + if (is_trigger_rule(parser, node.rule)) { + return LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL_MARKER; + } + + return std::nullopt; +} + +auto chat_input_of(const char * text, size_t text_length, const llama_token * byte_tokens) -> common_chat_input { + common_chat_input input; + input.text.assign(text, text_length); + const gsl::span tokens(byte_tokens, text_length); + input.tokens.assign(tokens.begin(), tokens.end()); + + return input; +} + +auto session_parser(const common_chat_session & session) -> const common_peg_arena & { + static const common_peg_arena content_only = + build_chat_peg_parser([](common_chat_peg_builder & builder) -> common_peg_parser { + return builder.content(builder.rest()) + builder.end(); + }); + + return session.parser().empty() ? content_only : session.parser(); +} +} // namespace + +extern "C" auto llama_rs_chat_session_create( + llama_rs_chat_templates_handle templates, + const struct llama_vocab * vocab, + const char * messages_json, + const char * tools_json, + llama_rs_chat_session_options options, + llama_rs_chat_session_handle * out_session, + char ** out_error) -> llama_rs_chat_session_create_status { + if (out_session != nullptr) { + *out_session = nullptr; + } + if (out_error != nullptr) { + *out_error = nullptr; + } + if (templates == nullptr) { + return LLAMA_RS_CHAT_SESSION_CREATE_NULL_TEMPLATES_ARG; + } + if (vocab == nullptr) { + return LLAMA_RS_CHAT_SESSION_CREATE_NULL_VOCAB_ARG; + } + if (messages_json == nullptr) { + return LLAMA_RS_CHAT_SESSION_CREATE_NULL_MESSAGES_JSON_ARG; + } + if (tools_json == nullptr) { + return LLAMA_RS_CHAT_SESSION_CREATE_NULL_TOOLS_JSON_ARG; + } + if (out_session == nullptr) { + return LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_SESSION_ARG; + } + if (out_error == nullptr) { + return LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_ERROR_ARG; + } + + try { + common_chat_templates_inputs inputs; + + try { + inputs.messages = common_chat_msgs_parse_oaicompat(common_json::parse(messages_json)); + } catch (const common_json_error & err) { + return llama_rs_report_exception_message( + out_error, err.what(), LLAMA_RS_CHAT_SESSION_CREATE_MESSAGES_NOT_JSON, + LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED); + } catch (const std::invalid_argument & err) { + return llama_rs_report_exception_message( + out_error, err.what(), LLAMA_RS_CHAT_SESSION_CREATE_MESSAGES_INVALID, + LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED); + } + + try { + inputs.tools = common_chat_tools_parse_oaicompat(common_json::parse(tools_json)); + } catch (const common_json_error & err) { + return llama_rs_report_exception_message( + out_error, err.what(), LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_NOT_JSON, + LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED); + } catch (const std::runtime_error & err) { + return llama_rs_report_exception_message( + out_error, err.what(), LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_INVALID, + LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED); + } + + inputs.add_generation_prompt = options.assistant_turn == LLAMA_RS_CHAT_ASSISTANT_TURN_NEW; + inputs.continue_final_message = continuation_of(options.assistant_turn); + inputs.enable_thinking = options.enable_thinking; + inputs.parallel_tool_calls = options.parallel_tool_calls; + inputs.tool_choice = tool_choice_of(options.tool_choice); + inputs.reasoning_format = COMMON_REASONING_FORMAT_AUTO; + + auto chat_session = std::make_unique(); + + try { + chat_session->session = common_chat_session(templates->templates.get(), vocab, inputs); + } catch (const std::bad_alloc &) { + throw; + } catch (const std::exception & err) { + return llama_rs_report_exception_message( + out_error, err.what(), LLAMA_RS_CHAT_SESSION_CREATE_TEMPLATE_CANNOT_SERVE_REQUEST, + LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED); + } + + chat_session->generation_prompt = common_chat_input_tokenize(vocab, chat_session->session.generation_prompt()); + for (const llama_token token : chat_session->generation_prompt.tokens) { + if (token != LLAMA_TOKEN_NULL) { + chat_session->generation_prompt_tokens.push_back(token); + } + } + chat_session->session.apply_sampling(chat_session->sampling); + chat_session->preserved_tokens.assign( + chat_session->sampling.preserved_tokens.begin(), chat_session->sampling.preserved_tokens.end()); + + *out_session = chat_session.release(); + + return LLAMA_RS_CHAT_SESSION_CREATE_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_CHAT_SESSION_CREATE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_SESSION_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED); + } +} + +extern "C" auto llama_rs_chat_session_free( + llama_rs_chat_session_handle session, + char ** out_error) -> llama_rs_chat_session_free_status { + if (out_error != nullptr) { + *out_error = nullptr; + } + try { + const std::unique_ptr reclaimed(session); + + return LLAMA_RS_CHAT_SESSION_FREE_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_CHAT_SESSION_FREE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_SESSION_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, + LLAMA_RS_CHAT_SESSION_FREE_ERROR_STRING_ALLOCATION_FAILED); + } +} + +extern "C" auto llama_rs_chat_session_prompt_part_count(llama_rs_chat_session_handle session) -> size_t { + return session->session.prompt().parts.size(); +} + +extern "C" auto llama_rs_chat_session_prompt_part_text( + llama_rs_chat_session_handle session, + size_t index, + size_t * out_length) -> const char * { + return llama_rs_string_view(session->session.prompt().parts.at(index).text, out_length); +} + +extern "C" auto llama_rs_chat_session_prompt_part_is_input(llama_rs_chat_session_handle session, size_t index) + -> bool { + return session->session.prompt().parts.at(index).is_input; +} + +extern "C" auto llama_rs_chat_session_grammar(llama_rs_chat_session_handle session, size_t * out_length) + -> const char * { + return llama_rs_string_view(session->session.grammar(), out_length); +} + +extern "C" auto llama_rs_chat_session_grammar_lazy(llama_rs_chat_session_handle session) -> bool { + return session->sampling.grammar_lazy; +} + +extern "C" auto llama_rs_chat_session_grammar_trigger_count(llama_rs_chat_session_handle session) -> size_t { + return session->sampling.grammar_triggers.size(); +} + +extern "C" auto llama_rs_chat_session_grammar_trigger_type(llama_rs_chat_session_handle session, size_t index) + -> llama_rs_chat_grammar_trigger_type { + switch (session->sampling.grammar_triggers.at(index).type) { + case COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN: + return LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_TOKEN; + case COMMON_GRAMMAR_TRIGGER_TYPE_WORD: + return LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_WORD; + case COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN: + return LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN; + case COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL: + return LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL; + } + + return LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL; +} + +extern "C" auto llama_rs_chat_session_grammar_trigger_value( + llama_rs_chat_session_handle session, + size_t index, + size_t * out_length) -> const char * { + return llama_rs_string_view(session->sampling.grammar_triggers.at(index).value, out_length); +} + +extern "C" auto llama_rs_chat_session_grammar_trigger_token(llama_rs_chat_session_handle session, size_t index) + -> llama_token { + return session->sampling.grammar_triggers.at(index).token; +} + +extern "C" auto llama_rs_chat_session_preserved_token_count(llama_rs_chat_session_handle session) -> size_t { + return session->preserved_tokens.size(); +} + +extern "C" auto llama_rs_chat_session_preserved_token(llama_rs_chat_session_handle session, size_t index) + -> llama_token { + return session->preserved_tokens.at(index); +} + +extern "C" auto llama_rs_chat_session_generation_prompt(llama_rs_chat_session_handle session, size_t * out_length) + -> const char * { + return llama_rs_string_view(session->session.generation_prompt(), out_length); +} + +extern "C" auto llama_rs_chat_session_generation_prompt_token_count(llama_rs_chat_session_handle session) -> size_t { + return session->generation_prompt_tokens.size(); +} + +extern "C" auto llama_rs_chat_session_generation_prompt_token(llama_rs_chat_session_handle session, size_t index) + -> llama_token { + return session->generation_prompt_tokens.at(index); +} + +extern "C" auto llama_rs_chat_session_thinking_start_tag(llama_rs_chat_session_handle session, size_t * out_length) + -> const char * { + return llama_rs_string_view(session->session.thinking_start_tag(), out_length); +} + +extern "C" auto llama_rs_chat_session_thinking_end_tag_count(llama_rs_chat_session_handle session) -> size_t { + return session->session.thinking_end_tags().size(); +} + +extern "C" auto llama_rs_chat_session_thinking_end_tag( + llama_rs_chat_session_handle session, + size_t index, + size_t * out_length) -> const char * { + return llama_rs_string_view(session->session.thinking_end_tags().at(index), out_length); +} + +extern "C" auto llama_rs_chat_session_additional_stop_count(llama_rs_chat_session_handle session) -> size_t { + return session->session.additional_stops().size(); +} + +extern "C" auto llama_rs_chat_session_additional_stop( + llama_rs_chat_session_handle session, + size_t index, + size_t * out_length) -> const char * { + return llama_rs_string_view(session->session.additional_stops().at(index), out_length); +} + +extern "C" auto llama_rs_chat_session_parse( + llama_rs_chat_session_handle session, + const char * text, + size_t text_length, + const llama_token * byte_tokens, + bool is_partial, + llama_rs_parsed_chat_handle * out_handle, + char ** out_error) -> llama_rs_chat_session_parse_status { + if (out_handle != nullptr) { + *out_handle = nullptr; + } + if (out_error != nullptr) { + *out_error = nullptr; + } + if (session == nullptr) { + return LLAMA_RS_CHAT_SESSION_PARSE_NULL_SESSION_ARG; + } + if (text == nullptr) { + return LLAMA_RS_CHAT_SESSION_PARSE_NULL_TEXT_ARG; + } + if (byte_tokens == nullptr) { + return LLAMA_RS_CHAT_SESSION_PARSE_NULL_BYTE_TOKENS_ARG; + } + if (out_handle == nullptr) { + return LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_HANDLE_ARG; + } + if (out_error == nullptr) { + return LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_ERROR_ARG; + } + + try { + common_chat_parser_params parser_params; + parser_params.format = session->session.format(); + parser_params.generation_prompt = session->generation_prompt; + + auto parsed = std::make_unique(); + + try { + parsed->message = common_chat_peg_parse( + session->session.parser(), chat_input_of(text, text_length, byte_tokens), is_partial, parser_params); + } catch (const std::bad_alloc &) { + throw; + } catch (const std::runtime_error & err) { + return llama_rs_report_exception_message( + out_error, err.what(), LLAMA_RS_CHAT_SESSION_PARSE_OUTPUT_DOES_NOT_MATCH_FORMAT, + LLAMA_RS_CHAT_SESSION_PARSE_ERROR_STRING_ALLOCATION_FAILED); + } + + *out_handle = parsed.release(); + + return LLAMA_RS_CHAT_SESSION_PARSE_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_CHAT_SESSION_PARSE_ERROR_STRING_ALLOCATION_FAILED); + } +} + +extern "C" auto llama_rs_chat_session_classify( + llama_rs_chat_session_handle session, + const char * text, + size_t text_length, + const llama_token * byte_tokens, + bool is_partial, + llama_rs_chat_spans_handle * out_spans, + char ** out_error) -> llama_rs_chat_session_classify_status { + if (out_spans != nullptr) { + *out_spans = nullptr; + } + if (out_error != nullptr) { + *out_error = nullptr; + } + if (session == nullptr) { + return LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_SESSION_ARG; + } + if (text == nullptr) { + return LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_TEXT_ARG; + } + if (byte_tokens == nullptr) { + return LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_BYTE_TOKENS_ARG; + } + if (out_spans == nullptr) { + return LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_SPANS_ARG; + } + if (out_error == nullptr) { + return LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_ERROR_ARG; + } + + try { + common_chat_input parse_input = chat_input_of(text, text_length, byte_tokens); + parse_input.prepend(session->generation_prompt); + + const size_t generation_prompt_length = session->generation_prompt.size(); + + common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_LENIENT; + if (!is_partial) { + flags |= COMMON_PEG_PARSE_FLAG_COMPLETE; + } + + common_peg_parse_context context(std::move(parse_input.text), std::move(parse_input.tokens), flags); + const common_peg_arena & parser = session_parser(session->session); + const common_peg_parse_result result = parser.parse(context); + + if (result.fail() && (!is_partial || result.end == 0)) { + return LLAMA_RS_CHAT_SESSION_CLASSIFY_OUTPUT_DOES_NOT_MATCH_FORMAT; + } + + auto spans = std::make_unique(); + spans->parsed_length = result.end > generation_prompt_length ? result.end - generation_prompt_length : 0; + + context.ast.visit(result, [&](const common_peg_ast_node & node) -> void { + const auto kind = span_kind_of_node(parser, node); + if (!kind || node.end <= generation_prompt_length) { + return; + } + spans->spans.push_back({ + *kind, + node.start > generation_prompt_length ? node.start - generation_prompt_length : 0, + node.end - generation_prompt_length, + }); + }); + + *out_spans = spans.release(); + + return LLAMA_RS_CHAT_SESSION_CLASSIFY_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_CHAT_SESSION_CLASSIFY_ERROR_STRING_ALLOCATION_FAILED); + } +} + +extern "C" auto llama_rs_chat_spans_parsed_length(llama_rs_chat_spans_handle spans) -> size_t { + return spans->parsed_length; +} + +extern "C" auto llama_rs_chat_spans_count(llama_rs_chat_spans_handle spans) -> size_t { + return spans->spans.size(); +} + +extern "C" auto llama_rs_chat_spans_kind(llama_rs_chat_spans_handle spans, size_t index) -> llama_rs_chat_span_kind { + return spans->spans.at(index).kind; +} + +extern "C" auto llama_rs_chat_spans_start(llama_rs_chat_spans_handle spans, size_t index) -> size_t { + return spans->spans.at(index).start; +} + +extern "C" auto llama_rs_chat_spans_end(llama_rs_chat_spans_handle spans, size_t index) -> size_t { + return spans->spans.at(index).end; +} + +extern "C" auto llama_rs_chat_spans_free( + llama_rs_chat_spans_handle spans, + char ** out_error) -> llama_rs_chat_spans_free_status { + if (out_error != nullptr) { + *out_error = nullptr; + } + try { + const std::unique_ptr reclaimed(spans); + + return LLAMA_RS_CHAT_SPANS_FREE_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_CHAT_SPANS_FREE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_SPANS_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, + LLAMA_RS_CHAT_SPANS_FREE_ERROR_STRING_ALLOCATION_FAILED); + } +} + +extern "C" auto llama_rs_regex_escape( + const char * text, + size_t text_length, + char ** out_string, + char ** out_error) -> llama_rs_regex_escape_status { + if (out_string != nullptr) { + *out_string = nullptr; + } + if (out_error != nullptr) { + *out_error = nullptr; + } + if (text == nullptr) { + return LLAMA_RS_REGEX_ESCAPE_NULL_TEXT_ARG; + } + if (out_string == nullptr) { + return LLAMA_RS_REGEX_ESCAPE_NULL_OUT_STRING_ARG; + } + if (out_error == nullptr) { + return LLAMA_RS_REGEX_ESCAPE_NULL_OUT_ERROR_ARG; + } + + try { + *out_string = llama_rs_dup_string(regex_escape(std::string(text, text_length))); + if (*out_string == nullptr) { + return LLAMA_RS_REGEX_ESCAPE_ERROR_STRING_ALLOCATION_FAILED; + } + + return LLAMA_RS_REGEX_ESCAPE_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_REGEX_ESCAPE_ERROR_STRING_ALLOCATION_FAILED); + } +} diff --git a/llama-cpp-bindings-sys/wrapper_chat_session.h b/llama-cpp-bindings-sys/wrapper_chat_session.h new file mode 100644 index 000000000..ee8f2c8e0 --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_chat_session.h @@ -0,0 +1,229 @@ +#pragma once + +#include "llama.cpp/include/llama.h" +#include "wrapper_parsed_chat.h" +#include "wrapper_chat_templates.h" + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct llama_rs_chat_session; +typedef struct llama_rs_chat_session * llama_rs_chat_session_handle; + +struct llama_rs_chat_spans; +typedef struct llama_rs_chat_spans * llama_rs_chat_spans_handle; + +typedef enum llama_rs_chat_tool_choice { + LLAMA_RS_CHAT_TOOL_CHOICE_AUTO = 0, + LLAMA_RS_CHAT_TOOL_CHOICE_NONE, + LLAMA_RS_CHAT_TOOL_CHOICE_REQUIRED, +} llama_rs_chat_tool_choice; + +typedef enum llama_rs_chat_assistant_turn { + LLAMA_RS_CHAT_ASSISTANT_TURN_OMITTED = 0, + LLAMA_RS_CHAT_ASSISTANT_TURN_NEW, + LLAMA_RS_CHAT_ASSISTANT_TURN_CONTINUE_REASONING, + LLAMA_RS_CHAT_ASSISTANT_TURN_CONTINUE_CONTENT, +} llama_rs_chat_assistant_turn; + +typedef struct llama_rs_chat_session_options { + llama_rs_chat_assistant_turn assistant_turn; + bool enable_thinking; + bool parallel_tool_calls; + llama_rs_chat_tool_choice tool_choice; +} llama_rs_chat_session_options; + +typedef enum llama_rs_chat_session_create_status { + LLAMA_RS_CHAT_SESSION_CREATE_OK = 0, + LLAMA_RS_CHAT_SESSION_CREATE_NULL_TEMPLATES_ARG, + LLAMA_RS_CHAT_SESSION_CREATE_NULL_VOCAB_ARG, + LLAMA_RS_CHAT_SESSION_CREATE_NULL_MESSAGES_JSON_ARG, + LLAMA_RS_CHAT_SESSION_CREATE_NULL_TOOLS_JSON_ARG, + LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_SESSION_ARG, + LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_ERROR_ARG, + LLAMA_RS_CHAT_SESSION_CREATE_MESSAGES_NOT_JSON, + LLAMA_RS_CHAT_SESSION_CREATE_MESSAGES_INVALID, + LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_NOT_JSON, + LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_INVALID, + LLAMA_RS_CHAT_SESSION_CREATE_TEMPLATE_CANNOT_SERVE_REQUEST, + LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_CHAT_SESSION_CREATE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_SESSION_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_chat_session_create_status; + +llama_rs_chat_session_create_status llama_rs_chat_session_create( + llama_rs_chat_templates_handle templates, + const struct llama_vocab * vocab, + const char * messages_json, + const char * tools_json, + llama_rs_chat_session_options options, + llama_rs_chat_session_handle * out_session, + char ** out_error); + +typedef enum llama_rs_chat_session_free_status { + LLAMA_RS_CHAT_SESSION_FREE_OK = 0, + LLAMA_RS_CHAT_SESSION_FREE_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_CHAT_SESSION_FREE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_SESSION_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, +} llama_rs_chat_session_free_status; + +llama_rs_chat_session_free_status llama_rs_chat_session_free( + llama_rs_chat_session_handle session, + char ** out_error); + +size_t llama_rs_chat_session_prompt_part_count(llama_rs_chat_session_handle session); + +const char * llama_rs_chat_session_prompt_part_text( + llama_rs_chat_session_handle session, + size_t index, + size_t * out_length); + +bool llama_rs_chat_session_prompt_part_is_input(llama_rs_chat_session_handle session, size_t index); + +const char * llama_rs_chat_session_grammar(llama_rs_chat_session_handle session, size_t * out_length); + +bool llama_rs_chat_session_grammar_lazy(llama_rs_chat_session_handle session); + +typedef enum llama_rs_chat_grammar_trigger_type { + LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_TOKEN = 0, + LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_WORD, + LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN, + LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL, +} llama_rs_chat_grammar_trigger_type; + +size_t llama_rs_chat_session_grammar_trigger_count(llama_rs_chat_session_handle session); + +llama_rs_chat_grammar_trigger_type llama_rs_chat_session_grammar_trigger_type( + llama_rs_chat_session_handle session, + size_t index); + +const char * llama_rs_chat_session_grammar_trigger_value( + llama_rs_chat_session_handle session, + size_t index, + size_t * out_length); + +llama_token llama_rs_chat_session_grammar_trigger_token(llama_rs_chat_session_handle session, size_t index); + +size_t llama_rs_chat_session_preserved_token_count(llama_rs_chat_session_handle session); + +llama_token llama_rs_chat_session_preserved_token(llama_rs_chat_session_handle session, size_t index); + +const char * llama_rs_chat_session_generation_prompt(llama_rs_chat_session_handle session, size_t * out_length); + +size_t llama_rs_chat_session_generation_prompt_token_count(llama_rs_chat_session_handle session); + +llama_token llama_rs_chat_session_generation_prompt_token(llama_rs_chat_session_handle session, size_t index); + +const char * llama_rs_chat_session_thinking_start_tag(llama_rs_chat_session_handle session, size_t * out_length); + +size_t llama_rs_chat_session_thinking_end_tag_count(llama_rs_chat_session_handle session); + +const char * llama_rs_chat_session_thinking_end_tag( + llama_rs_chat_session_handle session, + size_t index, + size_t * out_length); + +size_t llama_rs_chat_session_additional_stop_count(llama_rs_chat_session_handle session); + +const char * llama_rs_chat_session_additional_stop( + llama_rs_chat_session_handle session, + size_t index, + size_t * out_length); + +typedef enum llama_rs_chat_session_parse_status { + LLAMA_RS_CHAT_SESSION_PARSE_OK = 0, + LLAMA_RS_CHAT_SESSION_PARSE_NULL_SESSION_ARG, + LLAMA_RS_CHAT_SESSION_PARSE_NULL_TEXT_ARG, + LLAMA_RS_CHAT_SESSION_PARSE_NULL_BYTE_TOKENS_ARG, + LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_HANDLE_ARG, + LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_ERROR_ARG, + LLAMA_RS_CHAT_SESSION_PARSE_OUTPUT_DOES_NOT_MATCH_FORMAT, + LLAMA_RS_CHAT_SESSION_PARSE_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_chat_session_parse_status; + +llama_rs_chat_session_parse_status llama_rs_chat_session_parse( + llama_rs_chat_session_handle session, + const char * text, + size_t text_length, + const llama_token * byte_tokens, + bool is_partial, + llama_rs_parsed_chat_handle * out_handle, + char ** out_error); + +typedef enum llama_rs_chat_span_kind { + LLAMA_RS_CHAT_SPAN_KIND_REASONING_BLOCK = 0, + LLAMA_RS_CHAT_SPAN_KIND_REASONING, + LLAMA_RS_CHAT_SPAN_KIND_CONTENT, + LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL, + LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL_MARKER, +} llama_rs_chat_span_kind; + +typedef enum llama_rs_chat_session_classify_status { + LLAMA_RS_CHAT_SESSION_CLASSIFY_OK = 0, + LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_SESSION_ARG, + LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_TEXT_ARG, + LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_BYTE_TOKENS_ARG, + LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_SPANS_ARG, + LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_ERROR_ARG, + LLAMA_RS_CHAT_SESSION_CLASSIFY_OUTPUT_DOES_NOT_MATCH_FORMAT, + LLAMA_RS_CHAT_SESSION_CLASSIFY_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_chat_session_classify_status; + +llama_rs_chat_session_classify_status llama_rs_chat_session_classify( + llama_rs_chat_session_handle session, + const char * text, + size_t text_length, + const llama_token * byte_tokens, + bool is_partial, + llama_rs_chat_spans_handle * out_spans, + char ** out_error); + +size_t llama_rs_chat_spans_parsed_length(llama_rs_chat_spans_handle spans); + +size_t llama_rs_chat_spans_count(llama_rs_chat_spans_handle spans); + +llama_rs_chat_span_kind llama_rs_chat_spans_kind(llama_rs_chat_spans_handle spans, size_t index); + +size_t llama_rs_chat_spans_start(llama_rs_chat_spans_handle spans, size_t index); + +size_t llama_rs_chat_spans_end(llama_rs_chat_spans_handle spans, size_t index); + +typedef enum llama_rs_chat_spans_free_status { + LLAMA_RS_CHAT_SPANS_FREE_OK = 0, + LLAMA_RS_CHAT_SPANS_FREE_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_CHAT_SPANS_FREE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_SPANS_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, +} llama_rs_chat_spans_free_status; + +llama_rs_chat_spans_free_status llama_rs_chat_spans_free( + llama_rs_chat_spans_handle spans, + char ** out_error); + +typedef enum llama_rs_regex_escape_status { + LLAMA_RS_REGEX_ESCAPE_OK = 0, + LLAMA_RS_REGEX_ESCAPE_NULL_TEXT_ARG, + LLAMA_RS_REGEX_ESCAPE_NULL_OUT_STRING_ARG, + LLAMA_RS_REGEX_ESCAPE_NULL_OUT_ERROR_ARG, + LLAMA_RS_REGEX_ESCAPE_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_regex_escape_status; + +llama_rs_regex_escape_status llama_rs_regex_escape( + const char * text, + size_t text_length, + char ** out_string, + char ** out_error); + +#ifdef __cplusplus +} +#endif diff --git a/llama-cpp-bindings-sys/wrapper_chat_templates.cpp b/llama-cpp-bindings-sys/wrapper_chat_templates.cpp new file mode 100644 index 000000000..e86d80f64 --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_chat_templates.cpp @@ -0,0 +1,88 @@ +#include "wrapper_chat_templates.h" +#include "wrapper_chat_templates_handle.h" +#include "wrapper_report_current_exception.h" + +#include "llama.cpp/common/chat.h" +#include "llama.cpp/common/jinja/caps.h" +#include "llama.cpp/common/jinja/lexer.h" +#include "llama.cpp/common/jinja/parser.h" +#include "llama.cpp/common/jinja/runtime.h" +#include "llama.cpp/include/llama.h" + +#include +#include + +namespace { +auto template_supports_tool_calls(const std::string & source) -> bool { + jinja::lexer lexer; + jinja::program program = jinja::parse_from_tokens(lexer.tokenize(source)); + + return jinja::caps_get(program).supports_tool_calls; +} +} // namespace + +extern "C" auto llama_rs_chat_templates_init( + const struct llama_model * model, + llama_rs_chat_templates_handle * out_templates, + char ** out_error) -> llama_rs_chat_templates_init_status { + if (out_templates != nullptr) { + *out_templates = nullptr; + } + if (out_error != nullptr) { + *out_error = nullptr; + } + if (model == nullptr) { + return LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_MODEL_ARG; + } + if (out_templates == nullptr) { + return LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_TEMPLATES_ARG; + } + if (out_error == nullptr) { + return LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_ERROR_ARG; + } + if (llama_model_chat_template(model, nullptr) == nullptr) { + return LLAMA_RS_CHAT_TEMPLATES_INIT_MODEL_HAS_NO_CHAT_TEMPLATE; + } + + try { + auto templates = std::make_unique(); + templates->templates = common_chat_templates_init(model, std::string()); + + const std::string tool_use_source = common_chat_templates_source(templates->templates.get(), "tool_use"); + templates->supports_tool_calls = template_supports_tool_calls( + tool_use_source.empty() ? common_chat_templates_source(templates->templates.get()) : tool_use_source); + + *out_templates = templates.release(); + + return LLAMA_RS_CHAT_TEMPLATES_INIT_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_CHAT_TEMPLATES_INIT_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_TEMPLATES_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_CHAT_TEMPLATES_INIT_ERROR_STRING_ALLOCATION_FAILED); + } +} + +extern "C" auto llama_rs_chat_templates_free( + llama_rs_chat_templates_handle templates, + char ** out_error) -> llama_rs_chat_templates_free_status { + if (out_error != nullptr) { + *out_error = nullptr; + } + try { + const std::unique_ptr reclaimed(templates); + + return LLAMA_RS_CHAT_TEMPLATES_FREE_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_CHAT_TEMPLATES_FREE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_TEMPLATES_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, + LLAMA_RS_CHAT_TEMPLATES_FREE_ERROR_STRING_ALLOCATION_FAILED); + } +} + +extern "C" auto llama_rs_chat_templates_supports_tool_calls(llama_rs_chat_templates_handle templates) -> bool { + return templates->supports_tool_calls; +} diff --git a/llama-cpp-bindings-sys/wrapper_chat_templates.h b/llama-cpp-bindings-sys/wrapper_chat_templates.h new file mode 100644 index 000000000..c7a1b8f2e --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_chat_templates.h @@ -0,0 +1,45 @@ +#pragma once + +#include "llama.cpp/include/llama.h" + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct llama_rs_chat_templates; +typedef struct llama_rs_chat_templates * llama_rs_chat_templates_handle; + +typedef enum llama_rs_chat_templates_init_status { + LLAMA_RS_CHAT_TEMPLATES_INIT_OK = 0, + LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_MODEL_ARG, + LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_TEMPLATES_ARG, + LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_ERROR_ARG, + LLAMA_RS_CHAT_TEMPLATES_INIT_MODEL_HAS_NO_CHAT_TEMPLATE, + LLAMA_RS_CHAT_TEMPLATES_INIT_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_CHAT_TEMPLATES_INIT_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_TEMPLATES_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_chat_templates_init_status; + +llama_rs_chat_templates_init_status llama_rs_chat_templates_init( + const struct llama_model * model, + llama_rs_chat_templates_handle * out_templates, + char ** out_error); + +typedef enum llama_rs_chat_templates_free_status { + LLAMA_RS_CHAT_TEMPLATES_FREE_OK = 0, + LLAMA_RS_CHAT_TEMPLATES_FREE_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_CHAT_TEMPLATES_FREE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_TEMPLATES_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, +} llama_rs_chat_templates_free_status; + +llama_rs_chat_templates_free_status llama_rs_chat_templates_free( + llama_rs_chat_templates_handle templates, + char ** out_error); + +bool llama_rs_chat_templates_supports_tool_calls(llama_rs_chat_templates_handle templates); + +#ifdef __cplusplus +} +#endif diff --git a/llama-cpp-bindings-sys/wrapper_chat_templates_handle.h b/llama-cpp-bindings-sys/wrapper_chat_templates_handle.h new file mode 100644 index 000000000..eaf108db0 --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_chat_templates_handle.h @@ -0,0 +1,8 @@ +#pragma once + +#include "llama.cpp/common/chat.h" + +struct llama_rs_chat_templates { + common_chat_templates_ptr templates; + bool supports_tool_calls = false; +}; diff --git a/llama-cpp-bindings-sys/wrapper_parsed_chat.cpp b/llama-cpp-bindings-sys/wrapper_parsed_chat.cpp new file mode 100644 index 000000000..43a7b0dfc --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_parsed_chat.cpp @@ -0,0 +1,61 @@ +#include "wrapper_parsed_chat.h" +#include "wrapper_parsed_chat_handle.h" +#include "wrapper_report_current_exception.h" +#include "wrapper_utils.h" + +#include +#include + +extern "C" auto llama_rs_parsed_chat_free( + llama_rs_parsed_chat_handle handle, + char ** out_error) -> llama_rs_parsed_chat_free_status { + if (out_error != nullptr) { + *out_error = nullptr; + } + try { + const std::unique_ptr reclaimed(handle); + return LLAMA_RS_PARSED_CHAT_FREE_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_PARSED_CHAT_FREE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_PARSED_CHAT_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, + LLAMA_RS_PARSED_CHAT_FREE_ERROR_STRING_ALLOCATION_FAILED); + } +} + +extern "C" auto llama_rs_parsed_chat_content(llama_rs_parsed_chat_handle handle, size_t * out_length) + -> const char * { + return llama_rs_string_view(handle->message.content, out_length); +} + +extern "C" auto llama_rs_parsed_chat_reasoning_content( + llama_rs_parsed_chat_handle handle, + size_t * out_length) -> const char * { + return llama_rs_string_view(handle->message.reasoning_content, out_length); +} + +extern "C" auto llama_rs_parsed_chat_tool_call_count(llama_rs_parsed_chat_handle handle) -> size_t { + return handle->message.tool_calls.size(); +} + +extern "C" auto llama_rs_parsed_chat_tool_call_id( + llama_rs_parsed_chat_handle handle, + size_t index, + size_t * out_length) -> const char * { + return llama_rs_string_view(handle->message.tool_calls[index].id, out_length); +} + +extern "C" auto llama_rs_parsed_chat_tool_call_name( + llama_rs_parsed_chat_handle handle, + size_t index, + size_t * out_length) -> const char * { + return llama_rs_string_view(handle->message.tool_calls[index].name, out_length); +} + +extern "C" auto llama_rs_parsed_chat_tool_call_arguments( + llama_rs_parsed_chat_handle handle, + size_t index, + size_t * out_length) -> const char * { + return llama_rs_string_view(handle->message.tool_calls[index].arguments, out_length); +} diff --git a/llama-cpp-bindings-sys/wrapper_parsed_chat.h b/llama-cpp-bindings-sys/wrapper_parsed_chat.h new file mode 100644 index 000000000..1cc3e5a24 --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_parsed_chat.h @@ -0,0 +1,50 @@ +#pragma once + +#include "llama.cpp/include/llama.h" + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct llama_rs_parsed_chat; +typedef struct llama_rs_parsed_chat * llama_rs_parsed_chat_handle; + +typedef enum llama_rs_parsed_chat_free_status { + LLAMA_RS_PARSED_CHAT_FREE_OK = 0, + LLAMA_RS_PARSED_CHAT_FREE_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_PARSED_CHAT_FREE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_PARSED_CHAT_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, +} llama_rs_parsed_chat_free_status; + +llama_rs_parsed_chat_free_status llama_rs_parsed_chat_free( + llama_rs_parsed_chat_handle handle, + char ** out_error); + +const char * llama_rs_parsed_chat_content(llama_rs_parsed_chat_handle handle, size_t * out_length); + +const char * llama_rs_parsed_chat_reasoning_content( + llama_rs_parsed_chat_handle handle, + size_t * out_length); + +size_t llama_rs_parsed_chat_tool_call_count(llama_rs_parsed_chat_handle handle); + +const char * llama_rs_parsed_chat_tool_call_id( + llama_rs_parsed_chat_handle handle, + size_t index, + size_t * out_length); + +const char * llama_rs_parsed_chat_tool_call_name( + llama_rs_parsed_chat_handle handle, + size_t index, + size_t * out_length); + +const char * llama_rs_parsed_chat_tool_call_arguments( + llama_rs_parsed_chat_handle handle, + size_t index, + size_t * out_length); + +#ifdef __cplusplus +} +#endif diff --git a/llama-cpp-bindings-sys/wrapper_parsed_chat_handle.h b/llama-cpp-bindings-sys/wrapper_parsed_chat_handle.h new file mode 100644 index 000000000..976d9c647 --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_parsed_chat_handle.h @@ -0,0 +1,7 @@ +#pragma once + +#include "llama.cpp/common/chat.h" + +struct llama_rs_parsed_chat { + common_chat_msg message; +}; diff --git a/llama-cpp-bindings-sys/wrapper_reasoning.cpp b/llama-cpp-bindings-sys/wrapper_reasoning.cpp deleted file mode 100644 index ba0cb4f1f..000000000 --- a/llama-cpp-bindings-sys/wrapper_reasoning.cpp +++ /dev/null @@ -1,158 +0,0 @@ -#include "wrapper_reasoning.h" -#include "wrapper_token_text.h" - -#include "llama.cpp/common/chat-auto-parser.h" -#include "llama.cpp/common/chat.h" -#include "llama.cpp/common/json.h" -#include "llama.cpp/include/llama.h" -#include "wrapper_report_current_exception.h" - -#include -#include -#include -#include -#include - -using wrapper_helpers::token_text_or_empty; - -struct llama_rs_reasoning_markers { - std::string open; - std::vector closes; -}; - -namespace { - -auto find_reasoning_markers( - const common_chat_template & tmpl, - std::string * out_start, - std::vector * out_ends) -> bool { - autoparser::generation_params probe_params; - probe_params.add_generation_prompt = true; - probe_params.enable_thinking = true; - probe_params.is_inference = false; - probe_params.add_inference = false; - probe_params.mark_input = false; - probe_params.messages = common_json::array({ - common_json{ { "role", "user" }, { "content", "ping" } }, - }); - - if (auto specialized = common_chat_try_specialized_template(tmpl, probe_params)) { - if (specialized->supports_thinking - && !specialized->thinking_start_tag.empty() - && !specialized->thinking_end_tags.empty()) { - *out_start = std::move(specialized->thinking_start_tag); - *out_ends = std::move(specialized->thinking_end_tags); - return true; - } - } - - autoparser::autoparser parser; - parser.analyze_template(tmpl); - if (parser.reasoning.mode != autoparser::reasoning_mode::NONE - && !parser.reasoning.start.empty() - && !parser.reasoning.end.empty()) { - *out_start = std::move(parser.reasoning.start); - out_ends->push_back(std::move(parser.reasoning.end)); - return true; - } - - return false; -} - -} // namespace - -extern "C" auto llama_rs_detect_reasoning_markers( - const struct llama_model * model, - llama_rs_reasoning_markers ** out_markers, - char ** out_error) -> llama_rs_detect_reasoning_markers_status { - if (out_markers != nullptr) { - *out_markers = nullptr; - } - if (out_error != nullptr) { - *out_error = nullptr; - } - if (model == nullptr) { - return LLAMA_RS_DETECT_REASONING_MARKERS_NULL_MODEL_ARG; - } - if (out_markers == nullptr) { - return LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_MARKERS_ARG; - } - if (out_error == nullptr) { - return LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_ERROR_ARG; - } - - try { - const char * tmpl_src = llama_model_chat_template(model, nullptr); - if (tmpl_src == nullptr) { - return LLAMA_RS_DETECT_REASONING_MARKERS_MODEL_HAS_NO_CHAT_TEMPLATE; - } - - const llama_vocab * vocab = llama_model_get_vocab(model); - if (vocab == nullptr) { - return LLAMA_RS_DETECT_REASONING_MARKERS_MODEL_HAS_NO_VOCAB; - } - - std::string const bos_token = token_text_or_empty(vocab, llama_vocab_bos(vocab)); - std::string const eos_token = token_text_or_empty(vocab, llama_vocab_eos(vocab)); - - common_chat_template const tmpl(tmpl_src, bos_token, eos_token); - - auto detected = std::make_unique(); - if (!find_reasoning_markers(tmpl, &detected->open, &detected->closes)) { - return LLAMA_RS_DETECT_REASONING_MARKERS_OK; - } - - *out_markers = detected.release(); - - return LLAMA_RS_DETECT_REASONING_MARKERS_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_DETECT_REASONING_MARKERS_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_DETECT_REASONING_MARKERS_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_DETECT_REASONING_MARKERS_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_reasoning_markers_open( - const llama_rs_reasoning_markers * markers) -> const char * { - if (markers == nullptr) { - return nullptr; - } - return markers->open.c_str(); -} - -extern "C" auto llama_rs_reasoning_markers_close_count( - const llama_rs_reasoning_markers * markers) -> size_t { - if (markers == nullptr) { - return 0; - } - return markers->closes.size(); -} - -extern "C" auto llama_rs_reasoning_markers_close_at( - const llama_rs_reasoning_markers * markers, - size_t index) -> const char * { - if (markers == nullptr || index >= markers->closes.size()) { - return nullptr; - } - return markers->closes[index].c_str(); -} - -extern "C" auto llama_rs_reasoning_markers_free( - llama_rs_reasoning_markers * markers, - char ** out_error) -> llama_rs_reasoning_markers_free_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - try { - const std::unique_ptr reclaimed(markers); - return LLAMA_RS_REASONING_MARKERS_FREE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_REASONING_MARKERS_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_REASONING_MARKERS_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - LLAMA_RS_REASONING_MARKERS_FREE_ERROR_STRING_ALLOCATION_FAILED); - } -} diff --git a/llama-cpp-bindings-sys/wrapper_reasoning.h b/llama-cpp-bindings-sys/wrapper_reasoning.h deleted file mode 100644 index 9370c4323..000000000 --- a/llama-cpp-bindings-sys/wrapper_reasoning.h +++ /dev/null @@ -1,49 +0,0 @@ -#pragma once - -#include "llama.cpp/include/llama.h" -#include "wrapper_utils.h" - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -typedef enum llama_rs_detect_reasoning_markers_status { - LLAMA_RS_DETECT_REASONING_MARKERS_OK = 0, - LLAMA_RS_DETECT_REASONING_MARKERS_NULL_MODEL_ARG, - LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_MARKERS_ARG, - LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_ERROR_ARG, - LLAMA_RS_DETECT_REASONING_MARKERS_MODEL_HAS_NO_CHAT_TEMPLATE, - LLAMA_RS_DETECT_REASONING_MARKERS_MODEL_HAS_NO_VOCAB, - LLAMA_RS_DETECT_REASONING_MARKERS_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_DETECT_REASONING_MARKERS_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_DETECT_REASONING_MARKERS_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_detect_reasoning_markers_status; - -typedef struct llama_rs_reasoning_markers llama_rs_reasoning_markers; - -llama_rs_detect_reasoning_markers_status llama_rs_detect_reasoning_markers( - const struct llama_model * model, - llama_rs_reasoning_markers ** out_markers, - char ** out_error); - -const char * llama_rs_reasoning_markers_open(const llama_rs_reasoning_markers * markers); -size_t llama_rs_reasoning_markers_close_count(const llama_rs_reasoning_markers * markers); -const char * llama_rs_reasoning_markers_close_at( - const llama_rs_reasoning_markers * markers, - size_t index); -typedef enum llama_rs_reasoning_markers_free_status { - LLAMA_RS_REASONING_MARKERS_FREE_OK = 0, - LLAMA_RS_REASONING_MARKERS_FREE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_REASONING_MARKERS_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_REASONING_MARKERS_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, -} llama_rs_reasoning_markers_free_status; - -llama_rs_reasoning_markers_free_status llama_rs_reasoning_markers_free( - llama_rs_reasoning_markers * markers, - char ** out_error); - -#ifdef __cplusplus -} -#endif diff --git a/llama-cpp-bindings-sys/wrapper_token_text.cpp b/llama-cpp-bindings-sys/wrapper_token_text.cpp deleted file mode 100644 index 7719e1859..000000000 --- a/llama-cpp-bindings-sys/wrapper_token_text.cpp +++ /dev/null @@ -1,20 +0,0 @@ -#include "wrapper_token_text.h" -#include "llama.h" -#include - -namespace wrapper_helpers { - -auto token_text_or_empty(const llama_vocab * vocab, llama_token token) -> std::string { - if (token == LLAMA_TOKEN_NULL) { - return {}; - } - - const char * text = llama_vocab_get_text(vocab, token); - if (text == nullptr) { - return {}; - } - - return {text}; -} - -} // namespace wrapper_helpers diff --git a/llama-cpp-bindings-sys/wrapper_token_text.h b/llama-cpp-bindings-sys/wrapper_token_text.h deleted file mode 100644 index 231527e1a..000000000 --- a/llama-cpp-bindings-sys/wrapper_token_text.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#include "llama.cpp/include/llama.h" - -#include - -namespace wrapper_helpers { - -std::string token_text_or_empty(const llama_vocab * vocab, llama_token token); - -} diff --git a/llama-cpp-bindings-sys/wrapper_tool_calls.cpp b/llama-cpp-bindings-sys/wrapper_tool_calls.cpp deleted file mode 100644 index c742e50f8..000000000 --- a/llama-cpp-bindings-sys/wrapper_tool_calls.cpp +++ /dev/null @@ -1,236 +0,0 @@ -#include "wrapper_tool_calls.h" -#include "wrapper_token_text.h" - -#include "llama.cpp/common/chat-auto-parser.h" -#include "llama.cpp/common/chat-auto-parser-helpers.h" -#include "llama.cpp/common/chat.h" -#include "llama.cpp/common/json.h" -#include "llama.cpp/include/llama.h" -#include "wrapper_report_current_exception.h" -#include "wrapper_utils.h" - -#include -#include - -using wrapper_helpers::token_text_or_empty; - -namespace { - -struct tool_call_probe_params { - template_params without_tool_calls; - template_params with_tool_calls; -}; - -auto build_tool_call_probe_params() -> tool_call_probe_params { - common_json const user_msg = { - { "role", "user" }, - { "content", "Please use the tool" } - }; - common_json const assistant_no_tools = { - { "role", "assistant" }, - { "content", "Sure, calling." } - }; - common_json const first_tool_call = { - { "id", "call_001" }, - { "type", "function" }, - { "function", { - { "name", "tool_first" }, - { "arguments", { - { "arg_first", "XXXX" }, - { "arg_second", "YYYY" }, - }} - }} - }; - common_json const assistant_with_tools = { - { "role", "assistant" }, - { "content", "" }, - { "tool_calls", common_json::array({ first_tool_call }) } - }; - common_json const tool_definition = { - { "type", "function" }, - { "function", { - { "name", "tool_first" }, - { "description", "First test tool" }, - { "parameters", { - { "type", "object" }, - { "properties", { - { "arg_first", { { "type", "string" }, { "description", "first arg" } } }, - { "arg_second", { { "type", "string" }, { "description", "second arg" } } }, - }}, - { "required", common_json::array({ "arg_first", "arg_second" }) }, - }} - }} - }; - - tool_call_probe_params probe; - probe.without_tool_calls.messages = common_json::array({ user_msg, assistant_no_tools }); - probe.without_tool_calls.tools = common_json::array({ tool_definition }); - probe.without_tool_calls.add_generation_prompt = false; - probe.without_tool_calls.enable_thinking = true; - - probe.with_tool_calls = probe.without_tool_calls; - probe.with_tool_calls.messages = common_json::array({ user_msg, assistant_with_tools }); - - return probe; -} - -auto detect_tool_call_haystack( - const common_chat_template & tmpl, - const autoparser::analyze_reasoning & reasoning) -> std::string { - tool_call_probe_params const probe = build_tool_call_probe_params(); - - std::string const output_no_tools = autoparser::apply_template(tmpl, probe.without_tool_calls); - std::string const output_with_tools = autoparser::apply_template(tmpl, probe.with_tool_calls); - - if (output_no_tools.empty() || output_with_tools.empty()) { - return {}; - } - - diff_split const diff = calculate_diff_split(output_no_tools, output_with_tools); - std::string haystack = diff.right; - - auto remove_first = [&haystack](const std::string & needle) -> void { - if (needle.empty()) { - return; - } - auto pos = haystack.find(needle); - if (pos != std::string::npos) { - haystack.erase(pos, needle.length()); - } - }; - - remove_first(reasoning.start); - remove_first(reasoning.end); - - return haystack; -} - -} // namespace - -extern "C" auto llama_rs_compute_tool_call_haystack( - const struct llama_model * model, - char ** out_haystack, - char ** out_error) -> llama_rs_compute_tool_call_haystack_status { - if (out_haystack != nullptr) { - *out_haystack = nullptr; - } - if (out_error != nullptr) { - *out_error = nullptr; - } - if (model == nullptr) { - return LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_MODEL_ARG; - } - if (out_haystack == nullptr) { - return LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_OUT_HAYSTACK_ARG; - } - if (out_error == nullptr) { - return LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_OUT_ERROR_ARG; - } - - try { - const char * tmpl_src = llama_model_chat_template(model, nullptr); - if (tmpl_src == nullptr) { - return LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_MODEL_HAS_NO_CHAT_TEMPLATE; - } - - const llama_vocab * vocab = llama_model_get_vocab(model); - if (vocab == nullptr) { - return LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_MODEL_HAS_NO_VOCAB; - } - - std::string const bos_token = token_text_or_empty(vocab, llama_vocab_bos(vocab)); - std::string const eos_token = token_text_or_empty(vocab, llama_vocab_eos(vocab)); - - common_chat_template const tmpl(tmpl_src, bos_token, eos_token); - auto jinja_caps = tmpl.original_caps(); - autoparser::analyze_reasoning const reasoning(tmpl, jinja_caps.supports_tool_calls); - - std::string const haystack = detect_tool_call_haystack(tmpl, reasoning); - if (haystack.empty()) { - return LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_OK; - } - - char * haystack_dup = llama_rs_dup_string(haystack); - if (haystack_dup == nullptr) { - return LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_ERROR_STRING_ALLOCATION_FAILED; - } - - *out_haystack = haystack_dup; - - return LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_diagnose_tool_call_synthetic_renders( - const struct llama_model * model, - char ** out_no_tools, - char ** out_with_tools, - char ** out_error) -> llama_rs_diagnose_tool_call_synthetic_renders_status { - if (out_no_tools != nullptr) { - *out_no_tools = nullptr; - } - if (out_with_tools != nullptr) { - *out_with_tools = nullptr; - } - if (out_error != nullptr) { - *out_error = nullptr; - } - if (model == nullptr) { - return LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_MODEL_ARG; - } - if (out_no_tools == nullptr) { - return LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_NO_TOOLS_ARG; - } - if (out_with_tools == nullptr) { - return LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_WITH_TOOLS_ARG; - } - if (out_error == nullptr) { - return LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_ERROR_ARG; - } - - try { - const char * tmpl_src = llama_model_chat_template(model, nullptr); - if (tmpl_src == nullptr) { - return LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_MODEL_HAS_NO_CHAT_TEMPLATE; - } - - const llama_vocab * vocab = llama_model_get_vocab(model); - if (vocab == nullptr) { - return LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_MODEL_HAS_NO_VOCAB; - } - - std::string const bos_token = token_text_or_empty(vocab, llama_vocab_bos(vocab)); - std::string const eos_token = token_text_or_empty(vocab, llama_vocab_eos(vocab)); - - common_chat_template const tmpl(tmpl_src, bos_token, eos_token); - - tool_call_probe_params const probe = build_tool_call_probe_params(); - - std::string const output_a = autoparser::apply_template(tmpl, probe.without_tool_calls); - std::string const output_b = autoparser::apply_template(tmpl, probe.with_tool_calls); - - std::unique_ptr a_dup(llama_rs_dup_string(output_a)); - std::unique_ptr b_dup(llama_rs_dup_string(output_b)); - - if ((a_dup == nullptr) || (b_dup == nullptr)) { - return LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_ERROR_STRING_ALLOCATION_FAILED; - } - - *out_no_tools = a_dup.release(); - *out_with_tools = b_dup.release(); - - return LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_ERROR_STRING_ALLOCATION_FAILED); - } -} diff --git a/llama-cpp-bindings-sys/wrapper_tool_calls.h b/llama-cpp-bindings-sys/wrapper_tool_calls.h deleted file mode 100644 index fce2c366c..000000000 --- a/llama-cpp-bindings-sys/wrapper_tool_calls.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include "llama.cpp/include/llama.h" -#include "wrapper_utils.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef enum llama_rs_compute_tool_call_haystack_status { - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_OK = 0, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_MODEL_ARG, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_OUT_HAYSTACK_ARG, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_OUT_ERROR_ARG, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_MODEL_HAS_NO_CHAT_TEMPLATE, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_MODEL_HAS_NO_VOCAB, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_compute_tool_call_haystack_status; - -llama_rs_compute_tool_call_haystack_status llama_rs_compute_tool_call_haystack( - const struct llama_model * model, - char ** out_haystack, - char ** out_error); - -typedef enum llama_rs_diagnose_tool_call_synthetic_renders_status { - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_OK = 0, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_MODEL_ARG, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_NO_TOOLS_ARG, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_WITH_TOOLS_ARG, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_ERROR_ARG, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_MODEL_HAS_NO_CHAT_TEMPLATE, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_MODEL_HAS_NO_VOCAB, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_diagnose_tool_call_synthetic_renders_status; - -llama_rs_diagnose_tool_call_synthetic_renders_status llama_rs_diagnose_tool_call_synthetic_renders( - const struct llama_model * model, - char ** out_no_tools, - char ** out_with_tools, - char ** out_error); - -#ifdef __cplusplus -} -#endif diff --git a/llama-cpp-bindings-sys/wrapper_utils.h b/llama-cpp-bindings-sys/wrapper_utils.h index 43ee99ce6..7e45f1a3f 100644 --- a/llama-cpp-bindings-sys/wrapper_utils.h +++ b/llama-cpp-bindings-sys/wrapper_utils.h @@ -27,4 +27,9 @@ static inline char * llama_rs_dup_string(const char * value) noexcept { return llama_rs_dup_bytes(value, std::strlen(value)); } +static inline const char * llama_rs_string_view(const std::string & text, size_t * out_length) noexcept { + *out_length = text.size(); + return text.data(); +} + #endif diff --git a/llama-cpp-bindings-tests/src/build_user_prompt_with_media_marker.rs b/llama-cpp-bindings-tests/src/build_user_prompt_with_media_marker.rs deleted file mode 100644 index 2b3fabf7d..000000000 --- a/llama-cpp-bindings-tests/src/build_user_prompt_with_media_marker.rs +++ /dev/null @@ -1,16 +0,0 @@ -use anyhow::Result; -use llama_cpp_bindings::model::LlamaChatMessage; -use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::mtmd::mtmd_default_marker; - -/// # Errors -/// -/// Forwards chat-template lookup, message construction, and template application errors. -pub fn build_user_prompt_with_media_marker(model: &LlamaModel, question: &str) -> Result { - let marker = mtmd_default_marker()?; - let user_content = format!("{marker}{question}"); - let chat_template = model.chat_template(None)?; - let messages = [LlamaChatMessage::new("user".to_string(), user_content)?]; - - Ok(model.apply_chat_template(&chat_template, &messages, true, true)?) -} diff --git a/llama-cpp-bindings-tests/src/chat_answer.rs b/llama-cpp-bindings-tests/src/chat_answer.rs new file mode 100644 index 000000000..ebd7fd892 --- /dev/null +++ b/llama-cpp-bindings-tests/src/chat_answer.rs @@ -0,0 +1,126 @@ +use anyhow::Result; +use anyhow::bail; +use llama_cpp_bindings::ChatMessageParseOutcome; +use llama_cpp_bindings::ChatOutputCompleteness; +use llama_cpp_bindings::ChatSession; +use llama_cpp_bindings::TokenUsage; + +use crate::classify_sample_loop_outcome::ClassifySampleLoopOutcome; +use crate::recognized_answer::recognized_answer; + +const REASONING_PROBE: &str = "x"; + +fn thinking_tags(session: &ChatSession<'_>) -> Result> { + let mut tags = session.thinking_end_tags()?; + + tags.push(session.thinking_start_tag()?); + + Ok(tags) +} + +fn prompted_reasoning(session: &ChatSession<'_>) -> Result { + let probed = recognized_answer(session, REASONING_PROBE, ChatOutputCompleteness::Partial)?; + + if probed.content == REASONING_PROBE && probed.reasoning_content.is_empty() { + return Ok(String::new()); + } + + let Some(prompted) = probed.reasoning_content.strip_suffix(REASONING_PROBE) else { + bail!("the probe must continue either the prompt's reasoning or its content; {probed:?}"); + }; + + Ok(prompted.to_owned()) +} + +/// A generated answer to a chat session, with what the classifier attributed to each section. +#[derive(Debug)] +pub struct ChatAnswer { + pub outcome: ClassifySampleLoopOutcome, + pub usage: TokenUsage, +} + +impl ChatAnswer { + /// Asserts that every generated token is content, and that no thinking tag of `session` + /// reached the content stream. + /// + /// # Errors + /// + /// Forwards errors of reading the session's thinking tags. + /// + /// # Panics + /// + /// Panics when the answer reasons, holds a token of no section, or leaks a thinking tag. + pub fn assert_content_without_reasoning(&self, session: &ChatSession<'_>) -> Result<()> { + let Self { outcome, usage } = self; + + assert_ne!(outcome.generated.text(), "", "{outcome:?}"); + assert!(outcome.observed_content > 0, "{outcome:?}"); + assert_eq!(outcome.observed_reasoning, 0, "{outcome:?}"); + assert_eq!(outcome.observed_undeterminable, 0, "{outcome:?}"); + assert_eq!(usage.reasoning_tokens, 0, "{usage:?}"); + assert_eq!(usage.undeterminable_tokens, 0, "{usage:?}"); + assert_eq!( + usage.completion_tokens(), + outcome.observed_content, + "{usage:?}" + ); + + for tag in thinking_tags(session)? { + assert!( + !outcome.content_stream.contains(&tag), + "the content stream leaked the thinking tag {tag:?}; {outcome:?}" + ); + } + + Ok(()) + } + + /// Asserts that the answer reasons, and that the reasoning and content streams are exactly + /// what the session's parser extracts, apart from the reasoning text the generation prompt + /// itself holds, which the parser shows in front of the first generated text. + /// + /// # Errors + /// + /// Forwards errors of parsing the answer or reading the session's thinking tags, and fails + /// when the parser does not recognize the answer. + /// + /// # Panics + /// + /// Panics when the answer does not reason, holds a token of no section, leaks a thinking + /// tag, or streams text the parser does not extract. + pub fn assert_reasoning_matches_parse(&self, session: &ChatSession<'_>) -> Result<()> { + let Self { outcome, usage } = self; + let ChatMessageParseOutcome::Recognized(parsed) = + session.parse(&outcome.generated, ChatOutputCompleteness::Finished)? + else { + bail!("the session's parser must recognize the generated answer; {outcome:?}"); + }; + let prompted_reasoning = prompted_reasoning(session)?; + + assert!(outcome.observed_reasoning > 0, "{outcome:?}"); + assert_eq!(outcome.observed_undeterminable, 0, "{outcome:?}"); + assert_eq!( + usage.reasoning_tokens, outcome.observed_reasoning, + "{usage:?}" + ); + assert_eq!(usage.undeterminable_tokens, 0, "{usage:?}"); + assert_eq!( + usage.completion_tokens(), + outcome.observed_content + outcome.observed_reasoning, + "{usage:?}" + ); + assert_ne!(parsed.reasoning_content, "", "{outcome:?}"); + assert_eq!( + format!("{prompted_reasoning}{}", outcome.reasoning_stream), + parsed.reasoning_content + ); + assert_eq!(outcome.content_stream, parsed.content); + + for tag in thinking_tags(session)? { + assert!(!outcome.reasoning_stream.contains(&tag), "{outcome:?}"); + assert!(!outcome.content_stream.contains(&tag), "{outcome:?}"); + } + + Ok(()) + } +} diff --git a/llama-cpp-bindings-tests/src/chat_answer_generation.rs b/llama-cpp-bindings-tests/src/chat_answer_generation.rs new file mode 100644 index 000000000..65f9f01e6 --- /dev/null +++ b/llama-cpp-bindings-tests/src/chat_answer_generation.rs @@ -0,0 +1,58 @@ +use anyhow::Result; +use llama_cpp_bindings::BareJsonToolCalls; +use llama_cpp_bindings::ChatSession; +use llama_cpp_bindings::SampledTokenClassifier; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_test_harness::LlamaFixture; + +use crate::chat_answer::ChatAnswer; +use crate::classify_sample_loop::ClassifySampleLoop; + +/// Generates the answer to a chat session from its tokenized prompt, classifying every token. +pub struct ChatAnswerGeneration<'borrow, 'model> { + pub fixture: &'borrow LlamaFixture<'model>, + pub session: &'borrow ChatSession<'model>, + pub sampler: &'borrow mut LlamaSampler, + pub max_generated_tokens: i32, +} + +impl ChatAnswerGeneration<'_, '_> { + /// # Errors + /// + /// Forwards context creation, prompt tokenization, decoding, and sampling errors. + pub fn run(self) -> Result { + let Self { + fixture, + session, + sampler, + max_generated_tokens, + } = self; + let mut context = fixture.build_context()?; + let prompt_tokens = session.tokenize_prompt()?; + let mut classifier = + SampledTokenClassifier::for_chat_session(session, BareJsonToolCalls::Detect); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + + classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; + context.decode(&batch)?; + classifier.commit_prompt_tokens(); + + let initial_position = batch.n_tokens(); + let outcome = ClassifySampleLoop { + classifier: &mut classifier, + sampler, + context: &mut context, + batch: &mut batch, + initial_position, + max_generated_tokens, + } + .run()?; + + Ok(ChatAnswer { + outcome, + usage: classifier.into_usage(), + }) + } +} diff --git a/llama-cpp-bindings-tests/src/classify_sample_loop.rs b/llama-cpp-bindings-tests/src/classify_sample_loop.rs index d5949b182..3ce598942 100644 --- a/llama-cpp-bindings-tests/src/classify_sample_loop.rs +++ b/llama-cpp-bindings-tests/src/classify_sample_loop.rs @@ -1,15 +1,15 @@ use anyhow::Result; use llama_cpp_bindings::GenerationProgress; -use llama_cpp_bindings::SampledTokenSection; use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_bindings::ingest_outcome::IngestOutcome; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; -pub struct ClassifySampleLoop<'borrow, 'model> { - pub classifier: &'borrow mut SampledTokenClassifier<'model>, +use crate::classify_sample_loop_outcome::ClassifySampleLoopOutcome; + +pub struct ClassifySampleLoop<'borrow, 'session, 'model> { + pub classifier: &'borrow mut SampledTokenClassifier<'session, 'model>, pub sampler: &'borrow mut LlamaSampler, pub context: &'borrow mut LlamaContext<'model>, pub batch: &'borrow mut LlamaBatch, @@ -17,52 +17,10 @@ pub struct ClassifySampleLoop<'borrow, 'model> { pub max_generated_tokens: i32, } -#[derive(Debug, Default)] -pub struct ClassifySampleLoopOutcome { - pub generated_raw: String, - pub content_stream: String, - pub reasoning_stream: String, - pub observed_content: u64, - pub observed_reasoning: u64, - pub observed_tool_call: u64, - pub observed_undeterminable: u64, - pub eog_seen: bool, -} - -impl ClassifySampleLoopOutcome { - const fn record_end_of_generation(&mut self, section: SampledTokenSection) { - self.eog_seen = true; - - match section { - SampledTokenSection::Content => self.observed_content += 1, - SampledTokenSection::Reasoning => self.observed_reasoning += 1, - SampledTokenSection::ToolCall => self.observed_tool_call += 1, - SampledTokenSection::Pending => self.observed_undeterminable += 1, - } - } - - fn record_outcome(&mut self, ingest: &IngestOutcome) { - self.generated_raw.push_str(ingest.piece.raw()); - - match ingest.sampled_token { - SampledToken::Content(_) => { - self.observed_content += 1; - self.content_stream.push_str(ingest.piece.visible()); - } - SampledToken::Reasoning(_) => { - self.observed_reasoning += 1; - self.reasoning_stream.push_str(ingest.piece.visible()); - } - SampledToken::ToolCall(_) => self.observed_tool_call += 1, - SampledToken::Undeterminable(_) => self.observed_undeterminable += 1, - } - } -} - -impl ClassifySampleLoop<'_, '_> { +impl ClassifySampleLoop<'_, '_, '_> { /// # Errors - /// Forwards [`SampledTokenClassifier::sample`] / [`LlamaContext::decode`] / - /// [`LlamaBatch::add`] errors verbatim. Stops on the end of generation, on + /// Forwards [`SampledTokenClassifier::sample`] / [`SampledTokenClassifier::finish`] / + /// [`LlamaContext::decode`] / [`LlamaBatch::add`] errors verbatim. Stops on the end of generation, on /// `max_generated_tokens` exhaustion, or on the first error. pub fn run(self) -> Result { let mut outcome = ClassifySampleLoopOutcome::default(); @@ -92,7 +50,7 @@ impl ClassifySampleLoop<'_, '_> { self.context.decode(self.batch)?; } - self.classifier.finish(&mut ingest_outcomes); + self.classifier.finish(&mut ingest_outcomes)?; for ingest_outcome in &ingest_outcomes { outcome.record_outcome(ingest_outcome); @@ -101,65 +59,3 @@ impl ClassifySampleLoop<'_, '_> { Ok(outcome) } } - -#[cfg(test)] -mod tests { - use llama_cpp_bindings::SampledTokenSection; - use llama_cpp_bindings::TokenPiece; - use llama_cpp_bindings::ingest_outcome::IngestOutcome; - use llama_cpp_bindings::sampled_token::SampledToken; - use llama_cpp_bindings::token::LlamaToken; - - use super::ClassifySampleLoopOutcome; - - #[test] - fn records_a_tool_call_token_without_streaming_it() { - let mut outcome = ClassifySampleLoopOutcome::default(); - - outcome.record_outcome(&IngestOutcome { - sampled_token: SampledToken::ToolCall(LlamaToken(42)), - piece: TokenPiece::Visible("{".to_owned()), - }); - - assert_eq!(outcome.observed_tool_call, 1); - assert_eq!(outcome.generated_raw, "{"); - assert_eq!(outcome.content_stream, ""); - } - - #[test] - fn streams_the_visible_part_of_a_reasoning_token() { - let mut outcome = ClassifySampleLoopOutcome::default(); - - outcome.record_outcome(&IngestOutcome { - sampled_token: SampledToken::Reasoning(LlamaToken(7)), - piece: TokenPiece::Visible("thinking".to_owned()), - }); - - assert_eq!(outcome.observed_reasoning, 1); - assert_eq!(outcome.reasoning_stream, "thinking"); - } - - #[test] - fn counts_an_undeterminable_token_without_streaming_it() { - let mut outcome = ClassifySampleLoopOutcome::default(); - - outcome.record_outcome(&IngestOutcome { - sampled_token: SampledToken::Undeterminable(LlamaToken(9)), - piece: TokenPiece::Visible("ignored".to_owned()), - }); - - assert_eq!(outcome.observed_undeterminable, 1); - assert_eq!(outcome.content_stream, ""); - assert_eq!(outcome.reasoning_stream, ""); - } - - #[test] - fn counts_the_end_of_generation_in_the_section_it_ended() { - let mut outcome = ClassifySampleLoopOutcome::default(); - - outcome.record_end_of_generation(SampledTokenSection::Content); - - assert!(outcome.eog_seen); - assert_eq!(outcome.observed_content, 1); - } -} diff --git a/llama-cpp-bindings-tests/src/classify_sample_loop_outcome.rs b/llama-cpp-bindings-tests/src/classify_sample_loop_outcome.rs new file mode 100644 index 000000000..33101d91d --- /dev/null +++ b/llama-cpp-bindings-tests/src/classify_sample_loop_outcome.rs @@ -0,0 +1,109 @@ +use llama_cpp_bindings::ChatOutput; +use llama_cpp_bindings::SampledTokenSection; +use llama_cpp_bindings::ingest_outcome::IngestOutcome; +use llama_cpp_bindings::sampled_token::SampledToken; + +#[derive(Debug, Default)] +pub struct ClassifySampleLoopOutcome { + pub generated: ChatOutput, + pub content_stream: String, + pub reasoning_stream: String, + pub observed_content: u64, + pub observed_reasoning: u64, + pub observed_tool_call: u64, + pub observed_undeterminable: u64, + pub eog_seen: bool, +} + +impl ClassifySampleLoopOutcome { + pub const fn record_end_of_generation(&mut self, section: SampledTokenSection) { + self.eog_seen = true; + + match section { + SampledTokenSection::Content => self.observed_content += 1, + SampledTokenSection::Reasoning => self.observed_reasoning += 1, + SampledTokenSection::ToolCall => self.observed_tool_call += 1, + SampledTokenSection::Pending => self.observed_undeterminable += 1, + } + } + + pub fn record_outcome(&mut self, ingest: &IngestOutcome) { + self.generated + .push(ingest.sampled_token.token(), ingest.piece.raw()); + + match ingest.sampled_token { + SampledToken::Content(_) => { + self.observed_content += 1; + self.content_stream.push_str(ingest.piece.visible()); + } + SampledToken::Reasoning(_) => { + self.observed_reasoning += 1; + self.reasoning_stream.push_str(ingest.piece.visible()); + } + SampledToken::ToolCall(_) => self.observed_tool_call += 1, + SampledToken::Undeterminable(_) => self.observed_undeterminable += 1, + } + } +} + +#[cfg(test)] +mod tests { + use llama_cpp_bindings::SampledTokenSection; + use llama_cpp_bindings::TokenPiece; + use llama_cpp_bindings::ingest_outcome::IngestOutcome; + use llama_cpp_bindings::sampled_token::SampledToken; + use llama_cpp_bindings::token::LlamaToken; + + use super::ClassifySampleLoopOutcome; + + #[test] + fn records_a_tool_call_token_without_streaming_it() { + let mut outcome = ClassifySampleLoopOutcome::default(); + + outcome.record_outcome(&IngestOutcome { + sampled_token: SampledToken::ToolCall(LlamaToken(42)), + piece: TokenPiece::Visible("{".to_owned()), + }); + + assert_eq!(outcome.observed_tool_call, 1); + assert_eq!(outcome.generated.text(), "{"); + assert_eq!(outcome.content_stream, ""); + } + + #[test] + fn streams_the_visible_part_of_a_reasoning_token() { + let mut outcome = ClassifySampleLoopOutcome::default(); + + outcome.record_outcome(&IngestOutcome { + sampled_token: SampledToken::Reasoning(LlamaToken(7)), + piece: TokenPiece::Visible("thinking".to_owned()), + }); + + assert_eq!(outcome.observed_reasoning, 1); + assert_eq!(outcome.reasoning_stream, "thinking"); + } + + #[test] + fn counts_an_undeterminable_token_without_streaming_it() { + let mut outcome = ClassifySampleLoopOutcome::default(); + + outcome.record_outcome(&IngestOutcome { + sampled_token: SampledToken::Undeterminable(LlamaToken(9)), + piece: TokenPiece::Visible("ignored".to_owned()), + }); + + assert_eq!(outcome.observed_undeterminable, 1); + assert_eq!(outcome.content_stream, ""); + assert_eq!(outcome.reasoning_stream, ""); + } + + #[test] + fn counts_the_end_of_generation_in_the_section_it_ended() { + let mut outcome = ClassifySampleLoopOutcome::default(); + + outcome.record_end_of_generation(SampledTokenSection::Content); + + assert!(outcome.eog_seen); + assert_eq!(outcome.observed_content, 1); + } +} diff --git a/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs b/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs new file mode 100644 index 000000000..265b678d0 --- /dev/null +++ b/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs @@ -0,0 +1,43 @@ +use anyhow::Result; +use llama_cpp_bindings::ChatAssistantPrefill; +use llama_cpp_bindings::ChatAssistantTurn; +use llama_cpp_bindings::ChatSession; +use llama_cpp_bindings::ChatSessionOptions; +use llama_cpp_bindings::ChatToolChoice; +use llama_cpp_bindings::model::LlamaChatMessage; +use llama_cpp_bindings::model::LlamaModel; + +const PREFILLED_REASONING: &str = "\nThe question is simple, so I answer it directly.\n"; +const PREFILLED_CONTENT: &str = "\n\n"; + +/// A chat session for one user turn whose answer continues prefilled content. +/// +/// The prefill closes a reasoning block before the content begins, so the model writes only +/// content even when its template cannot disable thinking. +/// +/// # Errors +/// +/// Forwards message construction and chat session errors. +pub fn closed_reasoning_chat_session<'model>( + model: &'model LlamaModel, + question: &str, + tools_json: &str, +) -> Result> { + Ok(ChatSession::new( + model, + &[LlamaChatMessage::new( + "user".to_owned(), + question.to_owned(), + )?], + tools_json, + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::ContinueContent(ChatAssistantPrefill { + reasoning_content: PREFILLED_REASONING.to_owned(), + content: PREFILLED_CONTENT.to_owned(), + }), + enable_thinking: true, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + )?) +} diff --git a/llama-cpp-bindings-tests/src/get_weather_tools_json.rs b/llama-cpp-bindings-tests/src/get_weather_tools_json.rs new file mode 100644 index 000000000..71ce47531 --- /dev/null +++ b/llama-cpp-bindings-tests/src/get_weather_tools_json.rs @@ -0,0 +1,16 @@ +pub const GET_WEATHER_TOOLS_JSON: &str = r#"[ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "The city name"} + }, + "required": ["location"] + } + } + } +]"#; diff --git a/llama-cpp-bindings-tests/src/ingested_generation.rs b/llama-cpp-bindings-tests/src/ingested_generation.rs new file mode 100644 index 000000000..a4fea75e2 --- /dev/null +++ b/llama-cpp-bindings-tests/src/ingested_generation.rs @@ -0,0 +1,46 @@ +use anyhow::Context as _; +use anyhow::Result; +use llama_cpp_bindings::GenerationProgress; +use llama_cpp_bindings::SampledTokenClassifier; +use llama_cpp_bindings::ingest_outcome::IngestOutcome; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::LlamaModel; +use llama_cpp_bindings::model::ParseSpecialTokens; + +/// Feeds the tokens of `generated_text` to `classifier` as if the model generated them, then +/// ends the generation with the model's end-of-sequence token. +/// +/// # Errors +/// +/// Forwards tokenization and classification errors. +/// +/// # Panics +/// +/// Panics when the classifier ends the generation before the end-of-sequence token, or does +/// not end it on that token. +pub fn ingested_generation( + classifier: &mut SampledTokenClassifier<'_, '_>, + model: &LlamaModel, + generated_text: &str, +) -> Result> { + let mut outcomes = Vec::new(); + + for token in model.str_to_token(generated_text, AddBos::Never, ParseSpecialTokens::Always)? { + assert_eq!( + classifier.ingest(token, &mut outcomes)?, + GenerationProgress::Continues + ); + } + + assert_eq!( + classifier.ingest( + model + .token_eos() + .context("the model declares an EOS token")?, + &mut outcomes + )?, + GenerationProgress::Ended + ); + + Ok(outcomes) +} diff --git a/llama-cpp-bindings-tests/src/lib.rs b/llama-cpp-bindings-tests/src/lib.rs index 922330dd0..73ef8ce85 100644 --- a/llama-cpp-bindings-tests/src/lib.rs +++ b/llama-cpp-bindings-tests/src/lib.rs @@ -2,9 +2,21 @@ not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic) )] -pub mod build_user_prompt_with_media_marker; +pub mod chat_answer; +pub mod chat_answer_generation; pub mod chunk_token_breakdown; pub mod classify_sample_loop; +pub mod classify_sample_loop_outcome; +pub mod closed_reasoning_chat_session; pub mod fixtures_dir; +pub mod get_weather_tools_json; +pub mod ingested_generation; +pub mod media_question_chat_session; +pub mod normalize; +pub mod paris_weather_tool_call; +pub mod penalized_nucleus_sampler; pub mod prime_kv_cache; pub mod prime_kv_cache_with; +pub mod prompt_question; +pub mod recognized_answer; +pub mod user_turn_chat_session; diff --git a/llama-cpp-bindings-tests/src/media_question_chat_session.rs b/llama-cpp-bindings-tests/src/media_question_chat_session.rs new file mode 100644 index 000000000..951bb2c15 --- /dev/null +++ b/llama-cpp-bindings-tests/src/media_question_chat_session.rs @@ -0,0 +1,24 @@ +use anyhow::Result; +use llama_cpp_bindings::ChatSession; +use llama_cpp_bindings::model::LlamaModel; +use llama_cpp_bindings::mtmd::mtmd_default_marker; + +use crate::user_turn_chat_session::user_turn_chat_session; + +/// A chat session for one user turn that shows a medium and asks `question` about it, with +/// thinking enabled and no tools. +/// +/// # Errors +/// +/// Forwards message construction and chat session errors. +pub fn media_question_chat_session<'model>( + model: &'model LlamaModel, + question: &str, +) -> Result> { + user_turn_chat_session( + model, + &format!("{}{question}", mtmd_default_marker()?), + "[]", + true, + ) +} diff --git a/llama-cpp-bindings-tests/src/normalize.rs b/llama-cpp-bindings-tests/src/normalize.rs new file mode 100644 index 000000000..f8fd7695c --- /dev/null +++ b/llama-cpp-bindings-tests/src/normalize.rs @@ -0,0 +1,9 @@ +#[must_use] +pub fn normalize(input: &[f32]) -> Vec { + let magnitude = input + .iter() + .fold(0.0, |accumulator, &value| value.mul_add(value, accumulator)) + .sqrt(); + + input.iter().map(|&value| value / magnitude).collect() +} diff --git a/llama-cpp-bindings-tests/src/paris_weather_tool_call.rs b/llama-cpp-bindings-tests/src/paris_weather_tool_call.rs new file mode 100644 index 000000000..3afc20b8f --- /dev/null +++ b/llama-cpp-bindings-tests/src/paris_weather_tool_call.rs @@ -0,0 +1,14 @@ +use llama_cpp_bindings::ParsedToolCall; +use llama_cpp_bindings::ParsedToolCallId; +use llama_cpp_bindings::ToolCallArguments; +use serde_json::json; + +/// The `get_weather` call for Paris that the tool-call fixtures of every format encode. +#[must_use] +pub fn paris_weather_tool_call() -> ParsedToolCall { + ParsedToolCall::new( + ParsedToolCallId::NotProvidedByModel, + "get_weather".to_owned(), + ToolCallArguments::ValidJson(json!({"location": "Paris"})), + ) +} diff --git a/llama-cpp-bindings-tests/src/penalized_nucleus_sampler.rs b/llama-cpp-bindings-tests/src/penalized_nucleus_sampler.rs new file mode 100644 index 000000000..aef70ae1f --- /dev/null +++ b/llama-cpp-bindings-tests/src/penalized_nucleus_sampler.rs @@ -0,0 +1,22 @@ +use anyhow::Result; +use llama_cpp_bindings::model::LlamaModel; +use llama_cpp_bindings::sampling::LlamaSampler; + +const SAMPLING_SEED: u32 = 0x00C0_FFEE; + +/// A seeded sampler that penalizes repetition and samples from the nucleus, so a small model +/// does not loop while its answer stays reproducible. +/// +/// # Errors +/// +/// Forwards sampler construction errors. +pub fn penalized_nucleus_sampler(model: &LlamaModel) -> Result { + Ok(LlamaSampler::chain_simple([ + LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, + LlamaSampler::top_k(40)?, + LlamaSampler::top_p(0.9, 1)?, + LlamaSampler::min_p(0.05, 1)?, + LlamaSampler::temp(0.7)?, + LlamaSampler::dist(SAMPLING_SEED)?, + ])?) +} diff --git a/llama-cpp-bindings-tests/src/prompt_question.rs b/llama-cpp-bindings-tests/src/prompt_question.rs new file mode 100644 index 000000000..eb121df60 --- /dev/null +++ b/llama-cpp-bindings-tests/src/prompt_question.rs @@ -0,0 +1 @@ +pub const PROMPT_QUESTION: &str = "What animals do you see in this image?"; diff --git a/llama-cpp-bindings-tests/src/recognized_answer.rs b/llama-cpp-bindings-tests/src/recognized_answer.rs new file mode 100644 index 000000000..4b779bf9f --- /dev/null +++ b/llama-cpp-bindings-tests/src/recognized_answer.rs @@ -0,0 +1,25 @@ +use anyhow::Result; +use anyhow::bail; +use llama_cpp_bindings::ChatMessageParseOutcome; +use llama_cpp_bindings::ChatOutput; +use llama_cpp_bindings::ChatOutputCompleteness; +use llama_cpp_bindings::ChatSession; +use llama_cpp_bindings::ParsedChatMessage; + +/// Parses `answer` with the session's parser, failing when the parser does not recognize it. +/// +/// # Errors +/// +/// Forwards parse errors, and fails when the answer does not match the session's format. +pub fn recognized_answer( + session: &ChatSession<'_>, + answer: &str, + completeness: ChatOutputCompleteness, +) -> Result { + match session.parse(&ChatOutput::from(answer), completeness)? { + ChatMessageParseOutcome::Recognized(parsed) => Ok(parsed), + ChatMessageParseOutcome::Unrecognized(raw) => { + bail!("the session's parser must recognize {answer:?}; got {raw:?}") + } + } +} diff --git a/llama-cpp-bindings-tests/src/user_turn_chat_session.rs b/llama-cpp-bindings-tests/src/user_turn_chat_session.rs new file mode 100644 index 000000000..97b2ca2f8 --- /dev/null +++ b/llama-cpp-bindings-tests/src/user_turn_chat_session.rs @@ -0,0 +1,34 @@ +use anyhow::Result; +use llama_cpp_bindings::ChatAssistantTurn; +use llama_cpp_bindings::ChatSession; +use llama_cpp_bindings::ChatSessionOptions; +use llama_cpp_bindings::ChatToolChoice; +use llama_cpp_bindings::model::LlamaChatMessage; +use llama_cpp_bindings::model::LlamaModel; + +/// A chat session that answers one user turn, offering the tools of `tools_json`. +/// +/// # Errors +/// +/// Forwards message construction and chat session errors. +pub fn user_turn_chat_session<'model>( + model: &'model LlamaModel, + content: &str, + tools_json: &str, + enable_thinking: bool, +) -> Result> { + Ok(ChatSession::new( + model, + &[LlamaChatMessage::new( + "user".to_owned(), + content.to_owned(), + )?], + tools_json, + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::New, + enable_thinking, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + )?) +} diff --git a/llama-cpp-bindings-tests/tests/approximate_tok_env_eos_selection.rs b/llama-cpp-bindings-tests/tests/approximate_tok_env_eos_selection.rs new file mode 100644 index 000000000..46d2ac7d0 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/approximate_tok_env_eos_selection.rs @@ -0,0 +1,75 @@ +use anyhow::Context; +use anyhow::Result; +use llama_cpp_bindings::gguf_context::GgufContext; +use llama_cpp_bindings::model::LlamaModel; +use llama_cpp_bindings::model::params::LlamaModelParams; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; +use toktrie::INVALID_TOKEN; +use toktrie::TokenizerEnv as _; + +#[llama_test( + model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 64, + embeddings = true, +)] +fn approximate_tok_env_falls_back_to_eos_when_eot_unavailable( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let LlamaToken(eos) = fixture + .model + .token_eos() + .context("model declares an EOS token")?; + + assert_eq!( + fixture.model.approximate_tok_env()?.tok_trie().eos_token(), + eos.cast_unsigned() + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 64, + embeddings = true, +)] +fn approximate_tok_env_of_a_model_without_eot_and_eos_has_no_eos( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model_without_eos_path = std::env::temp_dir().join(format!( + "llama_cpp_bindings_{}_model_without_eos.gguf", + std::process::id() + )); + let mut gguf_context = GgufContext::from_file(fixture.model_path)?; + + gguf_context.remove_key("tokenizer.ggml.eos_token_id")?; + gguf_context.write_to_file(&model_without_eos_path)?; + + let model_without_eos = LlamaModel::load_from_file( + fixture.backend, + &model_without_eos_path, + &LlamaModelParams::default(), + )?; + + std::fs::remove_file(&model_without_eos_path)?; + + assert_eq!( + model_without_eos + .approximate_tok_env()? + .tok_trie() + .eos_token(), + INVALID_TOKEN + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/backend_initialization.rs b/llama-cpp-bindings-tests/tests/backend_initialization.rs index a71634acb..c4fcc4e33 100644 --- a/llama-cpp-bindings-tests/tests/backend_initialization.rs +++ b/llama-cpp-bindings-tests/tests/backend_initialization.rs @@ -1,4 +1,3 @@ -use anyhow::Result; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -11,11 +10,9 @@ use llama_cpp_test_harness::llama_test; n_ubatch = 64, void_logs = true, )] -fn void_logs_suppresses_output(fixture: &LlamaFixture<'_>) -> Result<()> { +fn void_logs_suppresses_output(fixture: &LlamaFixture<'_>) { assert!( fixture.model.n_vocab() > 0, "model must load successfully even when void_logs has been called before init" ); - - Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs b/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs new file mode 100644 index 000000000..4c0764579 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs @@ -0,0 +1,199 @@ +use anyhow::Result; +use anyhow::bail; +use llama_cpp_bindings::ChatMessageParseOutcome; +use llama_cpp_bindings::ChatOutput; +use llama_cpp_bindings::ChatOutputCompleteness; +use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; +use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_bindings_types::ParsedToolCall; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn parses_pure_content_response(fixture: &LlamaFixture<'_>) -> Result<()> { + let outcome = closed_reasoning_chat_session(fixture.model, "hello", "[]")?.parse( + &ChatOutput::from("hello world"), + ChatOutputCompleteness::Finished, + )?; + + let ChatMessageParseOutcome::Recognized(parsed) = outcome else { + bail!("expected Recognized for plain content; got Unrecognized"); + }; + assert_eq!(parsed.tool_calls, Vec::::new()); + assert!(!parsed.is_empty()); + assert!(parsed.content.contains("hello world")); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn parses_reasoning_section_into_reasoning_content(fixture: &LlamaFixture<'_>) -> Result<()> { + let input = "step one, step two\n\nactual response"; + let outcome = user_turn_chat_session(fixture.model, "hello", "[]", false)? + .parse(&ChatOutput::from(input), ChatOutputCompleteness::Finished)?; + + let ChatMessageParseOutcome::Recognized(parsed) = outcome else { + bail!("expected Recognized for reasoning section; got Unrecognized"); + }; + assert!( + parsed.reasoning_content.contains("step") || parsed.content.contains("step"), + "neither content nor reasoning contains 'step'; content={:?} reasoning={:?}", + parsed.content, + parsed.reasoning_content + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn parses_empty_input_yields_empty_message(fixture: &LlamaFixture<'_>) -> Result<()> { + let outcome = user_turn_chat_session(fixture.model, "hello", "[]", false)? + .parse(&ChatOutput::from(""), ChatOutputCompleteness::Finished)?; + + let ChatMessageParseOutcome::Recognized(parsed) = outcome else { + bail!("expected Recognized for empty input; got Unrecognized"); + }; + assert_eq!(parsed.tool_calls, Vec::::new()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn parses_an_answer_with_a_nul_byte_as_content(fixture: &LlamaFixture<'_>) -> Result<()> { + let outcome = user_turn_chat_session(fixture.model, "hello", "[]", false)?.parse( + &ChatOutput::from("hello\0world"), + ChatOutputCompleteness::Finished, + )?; + + let ChatMessageParseOutcome::Recognized(parsed) = outcome else { + bail!("an answer with a NUL byte must be recognized as content"); + }; + + assert_eq!(parsed.content, "hello\0world"); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_chat_session_parses_every_answer_it_is_given(fixture: &LlamaFixture<'_>) -> Result<()> { + let session = user_turn_chat_session(fixture.model, "hello", "[]", false)?; + + for answer in ["the first answer", "the second answer"] { + let ChatMessageParseOutcome::Recognized(parsed) = + session.parse(&ChatOutput::from(answer), ChatOutputCompleteness::Finished)? + else { + bail!("expected {answer:?} to be recognized as content"); + }; + + assert_eq!(parsed.content, answer); + } + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs b/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs new file mode 100644 index 000000000..e10969aa2 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs @@ -0,0 +1,109 @@ +use anyhow::Result; +use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace( + "unsloth/Ministral-3-14B-Reasoning-2512-GGUF", + "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf" + ), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn chat_session_renders_the_user_message_into_the_prompt(fixture: &LlamaFixture<'_>) -> Result<()> { + let prompt = user_turn_chat_session(fixture.model, "hello", "[]", true)?.prompt()?; + + assert!( + prompt.contains("hello"), + "the model's built-in chat template must render the user message content through the \ + engine; got: {prompt:?}" + ); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn chat_session_renders_long_messages_in_full(fixture: &LlamaFixture<'_>) -> Result<()> { + let long_content = "a".repeat(2000); + let prompt = user_turn_chat_session(fixture.model, &long_content, "[]", true)?.prompt()?; + + assert!( + prompt.contains(&long_content), + "a long user message must be rendered in full by the engine without truncation" + ); + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs new file mode 100644 index 000000000..f83e3e975 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs @@ -0,0 +1,58 @@ +use anyhow::Result; +use anyhow::bail; +use llama_cpp_bindings::ChatSessionError; +use llama_cpp_bindings::NewLlamaChatMessageError; +use llama_cpp_bindings_tests::media_question_chat_session::media_question_chat_session; +use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 64, + n_batch = 64, + n_ubatch = 64, +)] +fn media_question_chat_session_refuses_a_question_with_a_nul_byte( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let Err(failure) = media_question_chat_session(fixture.model, "describe\0this") else { + bail!("an interior NUL byte in the question must refuse the message"); + }; + let NewLlamaChatMessageError::NulError(nul_error) = + failure.downcast::()?; + + assert_eq!(nul_error.nul_position(), "<__media__>describe".len()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn chat_session_refuses_a_tool_missing_its_function_name(fixture: &LlamaFixture<'_>) -> Result<()> { + let tools_json = r#"[{"type":"function","function":{"description":"reports the weather"}}]"#; + let result = user_turn_chat_session(fixture.model, "hello", tools_json, false); + let Err(failure) = result else { + bail!("a tool without a function name must refuse the chat session"); + }; + + assert_eq!( + failure.downcast::()?, + ChatSessionError::ToolsInvalid { + message: "Failed to parse tools: [json.exception.out_of_range.403] key 'name' not \ + found; tools = [\n {\n \"type\": \"function\",\n \"function\": {\n \ + \"description\": \"reports the weather\"\n }\n }\n]" + .to_owned(), + } + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/chat_template_lookup.rs b/llama-cpp-bindings-tests/tests/chat_template_lookup.rs new file mode 100644 index 000000000..1215b8018 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/chat_template_lookup.rs @@ -0,0 +1,79 @@ +use llama_cpp_bindings::ChatTemplateError; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn chat_template_returns_non_empty(fixture: &LlamaFixture<'_>) { + let template = fixture.model.chat_template(None); + assert!(template.is_ok()); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn chat_template_with_nonexistent_name_returns_error(fixture: &LlamaFixture<'_>) { + let result = fixture + .model + .chat_template(Some("nonexistent_template_name_xyz")); + assert_eq!(result.unwrap_err(), ChatTemplateError::MissingTemplate); +} diff --git a/llama-cpp-bindings-tests/tests/chat_template_tool_call_support.rs b/llama-cpp-bindings-tests/tests/chat_template_tool_call_support.rs new file mode 100644 index 000000000..4d0fc27f3 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/chat_template_tool_call_support.rs @@ -0,0 +1,17 @@ +use anyhow::Result; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128 +)] +fn embedding_model_chat_template_renders_tool_calls(fixture: &LlamaFixture<'_>) -> Result<()> { + assert!(fixture.model.chat_templates()?.supports_tool_calls()); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs new file mode 100644 index 000000000..0407054a3 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs @@ -0,0 +1,301 @@ +use std::io::Write; +use std::time::Duration; + +use anyhow::Context as _; +use anyhow::Result; +use llama_cpp_bindings::BareJsonToolCalls; +use llama_cpp_bindings::ChatAssistantTurn; +use llama_cpp_bindings::ChatSession; +use llama_cpp_bindings::ChatSessionOptions; +use llama_cpp_bindings::ChatToolChoice; +use llama_cpp_bindings::SampledToken; +use llama_cpp_bindings::SampledTokenClassifier; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::ggml_time_us; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::LlamaChatMessage; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn raw_prompt_completion_usage_matches_the_classified_tokens( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let backend = fixture.backend; + let mut ctx = LlamaContext::from_model( + model, + backend, + (*fixture.context_params).into_llama_context_params(), + ) + .with_context(|| "unable to create context")?; + + let prompt = "Hello my name is"; + let max_generated_tokens: i32 = 64; + + let mut classifier = SampledTokenClassifier::undeterminable(model); + let tokens_list = model + .str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always) + .with_context(|| format!("failed to tokenize {prompt}"))?; + let prompt_token_count = u64::try_from(tokens_list.len())?; + + let mut decoder = encoding_rs::UTF_8.new_decoder(); + + for token in &tokens_list { + eprint!( + "{}", + model.token_to_piece(&SampledToken::Content(*token), &mut decoder, true, None)? + ); + } + std::io::stderr().flush()?; + + let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); + classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens_list, 0, false)?; + + assert_eq!(classifier.pending_prompt_tokens(), prompt_token_count); + assert_eq!(classifier.usage().prompt_tokens, 0); + + ctx.decode(&batch) + .with_context(|| "llama_decode() failed")?; + + let promoted = classifier.commit_prompt_tokens(); + assert_eq!(promoted, prompt_token_count); + assert_eq!(classifier.usage().prompt_tokens, prompt_token_count); + + let mut sampler = + LlamaSampler::chain_simple([LlamaSampler::dist(1234)?, LlamaSampler::greedy()?])?; + let initial_position = batch.n_tokens(); + let t_main_start = ggml_time_us(); + let outcome = ClassifySampleLoop { + classifier: &mut classifier, + sampler: &mut sampler, + context: &mut ctx, + batch: &mut batch, + initial_position, + max_generated_tokens, + } + .run()?; + let t_main_end = ggml_time_us(); + let duration = Duration::from_micros(u64::try_from(t_main_end - t_main_start)?); + let total_observed = + outcome.observed_content + outcome.observed_reasoning + outcome.observed_undeterminable; + + let tokens_per_second = f64::from(u32::try_from(total_observed)?) / duration.as_secs_f64(); + + eprintln!( + "\ndecoded {total_observed} tokens in {:.2} s, speed {tokens_per_second:.2} t/s", + duration.as_secs_f64(), + ); + + assert!( + !outcome.generated.text().is_empty(), + "model should generate at least one token" + ); + assert_eq!( + outcome.observed_tool_call, 0, + "raw prompt without tool-call markers must not produce ToolCall tokens; \ + outcome={outcome:?}" + ); + assert!( + total_observed > 0, + "model must produce at least one classified token; outcome={outcome:?}" + ); + + let usage = classifier.into_usage(); + assert_eq!( + usage.prompt_tokens, prompt_token_count, + "prompt_tokens must equal the tokenizer's prompt length" + ); + assert_eq!( + usage.content_tokens, outcome.observed_content, + "content_tokens must equal observed Content variants" + ); + assert_eq!( + usage.reasoning_tokens, outcome.observed_reasoning, + "reasoning_tokens must equal observed Reasoning variants" + ); + assert_eq!( + usage.undeterminable_tokens, outcome.observed_undeterminable, + "undeterminable_tokens must equal observed Undeterminable variants" + ); + assert_eq!( + usage.tool_call_tokens, outcome.observed_tool_call, + "tool_call_tokens must equal observed ToolCall variants" + ); + assert_eq!( + usage.completion_tokens(), + total_observed, + "completion_tokens must equal Content + Reasoning + Undeterminable" + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn chat_completion_usage_matches_the_classified_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let backend = fixture.backend; + let mut context = LlamaContext::from_model( + model, + backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let messages = [LlamaChatMessage::new( + "user".to_string(), + "Hello! How are you?".to_string(), + )?]; + let session = ChatSession::new( + model, + &messages, + "[]", + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::New, + enable_thinking: true, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + )?; + + let mut classifier = + SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let tokens = session.tokenize_prompt()?; + let prompt_token_count = u64::try_from(tokens.len())?; + + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; + + assert_eq!(classifier.pending_prompt_tokens(), prompt_token_count); + assert_eq!(classifier.usage().prompt_tokens, 0); + + context.decode(&batch)?; + + let promoted = classifier.commit_prompt_tokens(); + assert_eq!(promoted, prompt_token_count); + + let mut sampler = LlamaSampler::greedy()?; + let initial_position = batch.n_tokens(); + let outcome = ClassifySampleLoop { + classifier: &mut classifier, + sampler: &mut sampler, + context: &mut context, + batch: &mut batch, + initial_position, + max_generated_tokens: 512, + } + .run()?; + + println!(); + + assert!( + !outcome.generated.text().is_empty(), + "model should generate at least one token" + ); + let total_observed = + outcome.observed_content + outcome.observed_reasoning + outcome.observed_undeterminable; + assert!( + total_observed > 0, + "model must produce at least one classified token; outcome={outcome:?}" + ); + assert_eq!( + outcome.observed_tool_call, 0, + "chat without tool definitions must not produce ToolCall tokens; outcome={outcome:?}" + ); + + let usage = classifier.into_usage(); + + assert_eq!( + usage.prompt_tokens, prompt_token_count, + "prompt_tokens must equal the tokenizer's prompt length" + ); + assert_eq!( + usage.content_tokens, outcome.observed_content, + "content_tokens must equal observed Content variants" + ); + assert_eq!( + usage.reasoning_tokens, outcome.observed_reasoning, + "reasoning_tokens must equal observed Reasoning variants" + ); + assert_eq!( + usage.undeterminable_tokens, outcome.observed_undeterminable, + "undeterminable_tokens must equal observed Undeterminable variants" + ); + assert_eq!( + usage.completion_tokens(), + total_observed, + "completion_tokens must equal Content + Reasoning + Undeterminable" + ); + assert_eq!( + usage.tool_call_tokens, outcome.observed_tool_call, + "tool_call_tokens must equal observed ToolCall variants" + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/context_creation_limits.rs b/llama-cpp-bindings-tests/tests/context_creation_limits.rs new file mode 100644 index 000000000..cd5ff0c6f --- /dev/null +++ b/llama-cpp-bindings-tests/tests/context_creation_limits.rs @@ -0,0 +1,97 @@ +use std::num::NonZeroU32; + +use llama_cpp_bindings::LlamaContextLoadError; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 4294967295, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 4294967295, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 4294967295, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 4294967295, + n_batch = 128, + n_ubatch = 64, +)] +fn new_context_with_huge_ctx_returns_null_error(fixture: &LlamaFixture<'_>) { + let result = fixture.build_context(); + + assert!(result.is_err()); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 1, + n_ubatch = 1, +)] +fn new_context_rejects_more_sequences_than_one_batch_can_output(fixture: &LlamaFixture<'_>) { + let context_load = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params) + .into_llama_context_params() + .with_n_seq_max(2), + ); + + assert_eq!( + context_load.err(), + Some(LlamaContextLoadError::SequencesExceedOutputCapacity { + n_seq_max: 2, + output_capacity: 1, + }) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 2, + n_ubatch = 2, +)] +fn new_context_rejects_more_sequences_than_a_causal_context_can_output(fixture: &LlamaFixture<'_>) { + let context_load = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params) + .into_llama_context_params() + .with_n_ctx(NonZeroU32::new(1)) + .with_n_seq_max(2), + ); + + assert_eq!( + context_load.err(), + Some(LlamaContextLoadError::SequencesExceedOutputCapacity { + n_seq_max: 2, + output_capacity: 1, + }) + ); +} diff --git a/llama-cpp-bindings-tests/tests/context_dimensions.rs b/llama-cpp-bindings-tests/tests/context_dimensions.rs new file mode 100644 index 000000000..dfd672092 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/context_dimensions.rs @@ -0,0 +1,87 @@ +use anyhow::Result; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn new_context_returns_valid_context(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + assert!(context.n_ctx() > 0); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn context_reports_positive_context_and_batch_sizes(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + assert!(context.n_ctx() > 0); + assert!(context.n_batch() > 0); + assert!(context.n_ubatch() > 0); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 512, + n_seq_max = 4, +)] +fn n_ctx_seq_splits_the_context_between_sequences(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = fixture.build_context()?; + + assert_eq!(context.n_ctx(), 2048); + assert_eq!(context.n_ctx_seq(), 512); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/context_state.rs b/llama-cpp-bindings-tests/tests/context_state.rs deleted file mode 100644 index 1e498342c..000000000 --- a/llama-cpp-bindings-tests/tests/context_state.rs +++ /dev/null @@ -1,2804 +0,0 @@ -use std::num::NonZeroU8; -use std::num::NonZeroU32; -use std::sync::Arc; -use std::sync::atomic::AtomicBool; - -use anyhow::Result; -use anyhow::anyhow; -use llama_cpp_bindings::DecodeError; -use llama_cpp_bindings::EmbeddingsError; -use llama_cpp_bindings::LlamaContextLoadError; -use llama_cpp_bindings::LogitsError; -use llama_cpp_bindings::StringToTokenError; -use llama_cpp_bindings::batch_add_error::BatchAddError; -use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_bindings::context::context_limits::ContextLimits; -use llama_cpp_bindings::context::params::LlamaAttentionType; -use llama_cpp_bindings::context::params::LlamaPoolingType; -use llama_cpp_bindings::error::ClearKvCacheSeqError; -use llama_cpp_bindings::error::CopyKvCacheSeqError; -use llama_cpp_bindings::error::KvCacheSeqAddError; -use llama_cpp_bindings::error::KvCacheSeqDivError; -use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; -use llama_cpp_bindings::model::lora_adapter_scale::LoraAdapterScale; -use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_bindings::token::data::LlamaTokenData; -use llama_cpp_bindings::token::data_array::LlamaTokenDataArray; -use llama_cpp_bindings_tests::prime_kv_cache::prime_kv_cache; -use llama_cpp_bindings_tests::prime_kv_cache_with::prime_kv_cache_with; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::download_model::download_model; -use llama_cpp_test_harness::llama_test; - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn new_context_returns_valid_context(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - assert!(context.n_ctx() > 0); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4294967295, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4294967295, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4294967295, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4294967295, - n_batch = 128, - n_ubatch = 64, -)] -fn new_context_with_huge_ctx_returns_null_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = fixture.build_context(); - - assert!(result.is_err()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 1, - n_ubatch = 1, -)] -fn new_context_rejects_more_sequences_than_one_batch_can_output( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let context_load = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_n_seq_max(2), - ); - - assert_eq!( - context_load.err(), - Some(LlamaContextLoadError::SequencesExceedOutputCapacity { - n_seq_max: 2, - output_capacity: 1, - }) - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 2, - n_ubatch = 2, -)] -fn new_context_rejects_more_sequences_than_a_causal_context_can_output( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let context_load = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_n_ctx(NonZeroU32::new(1)) - .with_n_seq_max(2), - ); - - assert_eq!( - context_load.err(), - Some(LlamaContextLoadError::SequencesExceedOutputCapacity { - n_seq_max: 2, - output_capacity: 1, - }) - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn context_creation_and_properties(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - assert!(context.n_ctx() > 0); - assert!(context.n_batch() > 0); - assert!(context.n_ubatch() > 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 512, - n_seq_max = 4, -)] -fn n_ctx_seq_splits_the_context_between_sequences(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = fixture.build_context()?; - - assert_eq!(context.n_ctx(), 2048); - assert_eq!(context.n_ctx_seq(), 512); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn decode_and_get_logits(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - - let decode_result = context.decode(&batch); - assert!(decode_result.is_ok()); - - let logits = context.get_logits()?; - assert_ne!(logits, Vec::::new()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn timings_work(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - context.reset_timings(); - let timings = context.timings(); - assert!(timings.t_start_ms() >= 0.0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn token_data_array_has_entries_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let token_data_array = context.token_data_array()?; - - assert_ne!(token_data_array.data, Vec::::new()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn get_logits_ith_returns_valid_slice(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let last_index = i32::try_from(tokens.len() - 1)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let logits = context.get_logits_ith(last_index)?; - - assert_eq!(logits.len(), usize::try_from(fixture.model.n_vocab())?); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn token_data_array_ith_returns_valid_data(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let last_index = i32::try_from(tokens.len() - 1)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let token_data_array = context.token_data_array_ith(last_index)?; - - assert_eq!( - token_data_array.data.len(), - usize::try_from(fixture.model.n_vocab())? - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn embeddings_ith_returns_error_when_embeddings_disabled(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - assert_eq!(context.embeddings_ith(0), Err(EmbeddingsError::NotEnabled)); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn embeddings_seq_ith_returns_error_when_embeddings_disabled( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - assert_eq!( - context.embeddings_seq_ith(0), - Err(EmbeddingsError::NotEnabled) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn candidates_returns_n_vocab_entries(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let count = context.candidates()?.count(); - - assert_eq!(count, usize::try_from(fixture.model.n_vocab())?); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn debug_format_contains_struct_name(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let debug_output = format!("{context:?}"); - - assert!(debug_output.contains("LlamaContext")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn candidates_ith_returns_n_vocab_entries(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let last_index = i32::try_from(tokens.len() - 1)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let count = context.candidates_ith(last_index)?.count(); - - assert_eq!(count, usize::try_from(fixture.model.n_vocab())?); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace( - "Qwen/Qwen2.5-7B-Instruct-GGUF", - "qwen2.5-7b-instruct-q2_k.gguf" - ), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn set_lora_adapters_applies_a_real_adapter(fixture: &LlamaFixture<'_>) -> Result<()> { - let adapter_path = download_model( - "ggml-org/LoRA-Deepthink-Reasoning-Qwen2.5-7B-Instruct-Q8_0-GGUF", - "Deepthink-Reasoning-Adapter-q8_0.gguf", - )?; - let adapter = fixture.model.lora_adapter_init(&adapter_path)?; - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - assert_eq!( - context.set_lora_adapters(&[LoraAdapterScale { - adapter: &adapter, - scale: 1.0, - }]), - Ok(()) - ); - - assert_eq!(context.set_lora_adapters(&[]), Ok(())); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn set_lora_adapters_accepts_an_empty_adapter_set(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let result = context.set_lora_adapters(&[]); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn encode_on_non_encoder_model_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - - let result = context.encode(&batch); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - embeddings = true, -)] -fn embeddings_seq_ith_returns_null_embedding_error_for_invalid_seq( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_pooling_type(LlamaPoolingType::Last), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - assert_eq!( - context.embeddings_seq_ith(999), - Err(EmbeddingsError::SequenceEmbeddingUnavailable { - sequence_index: 999 - }) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn decode_empty_batch_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let batch = LlamaBatch::new(ContextLimits::of(&context)); - - let result = context.decode(&batch); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn set_abort_flag_aborts_decode(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let abort_flag = Arc::new(AtomicBool::new(true)); - context.set_abort_flag(abort_flag); - - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - - let result = context.decode(&batch); - - assert_eq!(result, Err(DecodeError::Aborted)); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn set_abort_flag_false_allows_decode(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let abort_flag = Arc::new(AtomicBool::new(false)); - context.set_abort_flag(abort_flag); - - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - - let result = context.decode(&batch); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn clear_abort_callback_allows_decode_with_flag_true(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let abort_flag = Arc::new(AtomicBool::new(true)); - context.set_abort_flag(abort_flag); - context.clear_abort_callback(); - - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - - let result = context.decode(&batch); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn synchronize_completes_without_panic(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - context.synchronize(); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn detach_threadpool_completes_without_panic(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - context.detach_threadpool(); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn get_logits_ith_returns_token_not_initialized_for_unknown_index( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let result = context.get_logits_ith(7); - - assert_eq!(result, Err(LogitsError::TokenNotInitialized(7))); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn clear_kv_cache_resets_positions(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - prime_kv_cache(fixture, &mut context)?; - - context.clear_kv_cache(); - assert_eq!(context.kv_cache_seq_pos_max(0)?, -1); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_pos_max_is_non_negative_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - prime_kv_cache(fixture, &mut context)?; - - assert!(context.kv_cache_seq_pos_max(0)? >= 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 256, - n_ubatch = 64, -)] -fn prime_kv_cache_surfaces_each_underlying_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - assert!( - prime_kv_cache_with(fixture, &mut context, "Hello\0world") - .is_err_and(|error| error.downcast_ref::().is_some()), - "an interior null byte must surface a tokenization error" - ); - assert_eq!( - prime_kv_cache_with(fixture, &mut context, &"word ".repeat(300)) - .err() - .and_then(|error| error.downcast_ref::().cloned()), - Some(BatchAddError::BatchFull { n_batch: 256 }), - "more tokens than one decode takes must surface an add-sequence error" - ); - - let filler = "word ".repeat(40); - let mut decode_result = prime_kv_cache_with(fixture, &mut context, &filler); - let mut attempts = 0; - while decode_result.is_ok() && attempts < 16 { - decode_result = prime_kv_cache_with(fixture, &mut context, &filler); - attempts += 1; - } - assert!( - decode_result.is_err(), - "filling the context past its window must surface a decode error" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn clear_kv_cache_seq_with_range(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - prime_kv_cache(fixture, &mut context)?; - - assert_eq!( - context.clear_kv_cache_seq(Some(0), Some(0), Some(1)), - Ok(()) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn copy_kv_cache_seq_succeeds(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - prime_kv_cache(fixture, &mut context)?; - - assert_eq!(context.copy_kv_cache_seq(0, 1, None, None), Ok(())); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_add_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - prime_kv_cache(fixture, &mut context)?; - - let result = context.kv_cache_seq_add(0, Some(0), None, 1); - - assert_eq!( - result.unwrap_err(), - KvCacheSeqAddError::IncompatibleRopeType - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_div_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - prime_kv_cache(fixture, &mut context)?; - - let divisor = NonZeroU8::new(2).ok_or_else(|| anyhow!("2 is non-zero"))?; - let result = context.kv_cache_seq_div(0, Some(0), None, divisor); - - assert_eq!( - result.unwrap_err(), - KvCacheSeqDivError::IncompatibleRopeType - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_keep_retains_specified_sequence(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - prime_kv_cache(fixture, &mut context)?; - - context.kv_cache_seq_keep(0); - - assert!(context.kv_cache_seq_pos_max(0)? >= 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn copy_kv_cache_seq_with_explicit_range(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - prime_kv_cache(fixture, &mut context)?; - - assert_eq!(context.copy_kv_cache_seq(0, 2, Some(0), Some(1)), Ok(())); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_pos_max_rejects_out_of_range_sequence(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = fixture.build_context()?; - - let result = context.kv_cache_seq_pos_max(999); - - assert_eq!( - result, - Err(llama_cpp_bindings::KvCacheSeqPosMaxError::SequenceIdOutOfRange { seq_id: 999 }) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn copy_kv_cache_seq_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let result = context.copy_kv_cache_seq(0, 1, Some(u32::MAX), None); - - assert_eq!( - result.unwrap_err(), - CopyKvCacheSeqError::P0TooLarge( - i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") - ) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn copy_kv_cache_seq_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let result = context.copy_kv_cache_seq(0, 1, Some(0), Some(u32::MAX)); - - assert_eq!( - result.unwrap_err(), - CopyKvCacheSeqError::P1TooLarge( - i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") - ) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn clear_kv_cache_seq_rejects_sequence_id_exceeding_i32_max( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = fixture.build_context()?; - - let result = context.clear_kv_cache_seq(Some(u32::MAX), None, None); - - assert_eq!( - result.unwrap_err(), - ClearKvCacheSeqError::SeqIdTooLarge( - i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") - ) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn clear_kv_cache_seq_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let result = context.clear_kv_cache_seq(Some(0), Some(u32::MAX), None); - - assert_eq!( - result.unwrap_err(), - ClearKvCacheSeqError::P0TooLarge( - i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") - ) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn clear_kv_cache_seq_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let result = context.clear_kv_cache_seq(Some(0), Some(0), Some(u32::MAX)); - - assert_eq!( - result.unwrap_err(), - ClearKvCacheSeqError::P1TooLarge( - i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") - ) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_add_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let result = context.kv_cache_seq_add(0, Some(u32::MAX), None, 1); - - assert_eq!( - result.unwrap_err(), - KvCacheSeqAddError::P0TooLarge( - i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") - ) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_add_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let result = context.kv_cache_seq_add(0, Some(0), Some(u32::MAX), 1); - - assert_eq!( - result.unwrap_err(), - KvCacheSeqAddError::P1TooLarge( - i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") - ) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_div_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let divisor = NonZeroU8::new(2).ok_or_else(|| anyhow!("2 is non-zero"))?; - let result = context.kv_cache_seq_div(0, Some(u32::MAX), None, divisor); - - assert_eq!( - result.unwrap_err(), - KvCacheSeqDivError::P0TooLarge( - i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") - ) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_div_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let divisor = NonZeroU8::new(2).ok_or_else(|| anyhow!("2 is non-zero"))?; - let result = context.kv_cache_seq_div(0, Some(0), Some(u32::MAX), divisor); - - assert_eq!( - result.unwrap_err(), - KvCacheSeqDivError::P1TooLarge( - i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") - ) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn save_and_load_session_file(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let tokens = - fixture - .model - .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let session_path = std::env::temp_dir().join("llama_test_session.bin"); - context.state_save_file(&session_path, &tokens)?; - - let loaded_tokens = context.state_load_file(&session_path, 512)?; - assert_eq!(loaded_tokens, tokens); - - std::fs::remove_file(&session_path)?; - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn get_state_size_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = fixture.build_context()?; - - assert!(context.get_state_size() > 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_save_and_load_file_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let tokens = - fixture - .model - .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let session_path = std::env::temp_dir().join("llama_test_seq_state.bin"); - let bytes_written = context.state_seq_save_file(&session_path, 0, &tokens)?; - assert!(bytes_written > 0); - - let loaded = context.state_seq_load_file(&session_path, 0, 512)?; - - assert_eq!(loaded.tokens, tokens); - assert_eq!( - loaded.bytes_read, bytes_written, - "loading must consume exactly the bytes the save produced" - ); - - std::fs::remove_file(&session_path)?; - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn set_state_data_rejects_a_truncated_snapshot(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let tokens = - fixture - .model - .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let mut state_data = vec![0u8; context.get_state_size()]; - let bytes_copied = unsafe { context.copy_state_data(&mut state_data) }?; - state_data.truncate(bytes_copied / 2); - - assert_eq!( - unsafe { context.set_state_data(&state_data) }, - Err( - llama_cpp_bindings::context::state_data_error::StateDataError::NothingRestored { - provided_bytes: state_data.len(), - } - ), - "a truncated snapshot restores nothing, and that must not be reported as success" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn copy_state_data_and_set_state_data_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let tokens = - fixture - .model - .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let state_size = context.get_state_size(); - let mut state_data = vec![0u8; state_size]; - let bytes_copied = unsafe { context.copy_state_data(&mut state_data) }?; - assert!(bytes_copied > 0); - - let bytes_read = unsafe { context.set_state_data(&state_data) }?; - assert_eq!( - bytes_read, bytes_copied, - "restoring the state must consume exactly the bytes the snapshot produced" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_load_file_with_nonexistent_file_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let result = context.state_load_file("/nonexistent/session.bin", 512); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_load_file_with_nonexistent_file_returns_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = fixture.build_context()?; - - let result = context.state_seq_load_file("/nonexistent/seq_state.bin", 0, 512); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_save_file_to_invalid_directory_returns_failed_to_save( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let context = fixture.build_context()?; - - let result = context.state_save_file("/nonexistent_dir/session.bin", &[]); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_save_file_to_invalid_directory_returns_failed_to_save( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let context = fixture.build_context()?; - - let result = context.state_seq_save_file("/nonexistent_dir/seq_state.bin", 0, &[]); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_load_file_with_zero_max_tokens_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let tokens = - fixture - .model - .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let session_path = std::env::temp_dir().join("llama_test_session_zero_max.bin"); - context.state_save_file(&session_path, &tokens)?; - - let result = context.state_load_file(&session_path, 0); - - assert!(result.is_err()); - let _ = std::fs::remove_file(&session_path); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_load_file_with_zero_max_tokens_returns_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = fixture.build_context()?; - - let tokens = - fixture - .model - .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let session_path = std::env::temp_dir().join("llama_test_seq_state_zero_max.bin"); - context.state_seq_save_file(&session_path, 0, &tokens)?; - - let result = context.state_seq_load_file(&session_path, 0, 0); - - assert!(result.is_err()); - let _ = std::fs::remove_file(&session_path); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_load_file_with_insufficient_max_tokens_returns_length_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = fixture.build_context()?; - - let tokens = fixture.model.str_to_token( - "Hello world this is a longer string for more tokens", - AddBos::Always, - ParseSpecialTokens::Always, - )?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let session_path = std::env::temp_dir().join("llama_test_session_insuf.bin"); - context.state_save_file(&session_path, &tokens)?; - - let result = context.state_load_file(&session_path, 1); - - assert!(result.is_err()); - let _ = std::fs::remove_file(&session_path); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_load_file_with_insufficient_max_tokens_returns_length_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = fixture.build_context()?; - - let tokens = fixture.model.str_to_token( - "Hello world this is a longer string for more tokens", - AddBos::Always, - ParseSpecialTokens::Always, - )?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let session_path = std::env::temp_dir().join("llama_test_seq_state_insuf.bin"); - context.state_seq_save_file(&session_path, 0, &tokens)?; - - let result = context.state_seq_load_file(&session_path, 0, 1); - - assert!(result.is_err()); - let _ = std::fs::remove_file(&session_path); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - - let context = fixture.build_context()?; - - let non_utf8_path = std::path::Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); - let result = context.state_save_file(non_utf8_path, &[]); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - - let mut context = fixture.build_context()?; - - let non_utf8_path = std::path::Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); - let result = context.state_load_file(non_utf8_path, 512); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - - let context = fixture.build_context()?; - - let non_utf8_path = std::path::Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); - let result = context.state_seq_save_file(non_utf8_path, 0, &[]); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - - let mut context = fixture.build_context()?; - - let non_utf8_path = std::path::Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); - let result = context.state_seq_load_file(non_utf8_path, 0, 512); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_save_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = fixture.build_context()?; - - let path_with_null = std::path::Path::new("/tmp/foo\0bar.bin"); - let result = context.state_save_file(path_with_null, &[]); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_load_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let path_with_null = std::path::Path::new("/tmp/foo\0bar.bin"); - let result = context.state_load_file(path_with_null, 512); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_save_file_with_null_byte_in_path_returns_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let context = fixture.build_context()?; - - let path_with_null = std::path::Path::new("/tmp/foo\0bar.bin"); - let result = context.state_seq_save_file(path_with_null, 0, &[]); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_load_file_with_null_byte_in_path_returns_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = fixture.build_context()?; - - let path_with_null = std::path::Path::new("/tmp/foo\0bar.bin"); - let result = context.state_seq_load_file(path_with_null, 0, 512); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_get_size_ext_returns_size_for_decoded_sequence( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - use llama_cpp_bindings::context::llama_state_seq_flags::LlamaStateSeqFlags; - - let mut context = fixture.build_context()?; - - let tokens = - fixture - .model - .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let flags = LlamaStateSeqFlags::empty(); - let size = context.state_seq_get_size_ext(0, &flags); - - assert!(size > 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_get_data_ext_and_set_data_ext_round_trip(fixture: &LlamaFixture<'_>) -> Result<()> { - use llama_cpp_bindings::context::llama_state_seq_flags::LlamaStateSeqFlags; - - let mut context = fixture.build_context()?; - - let tokens = - fixture - .model - .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let flags = LlamaStateSeqFlags::empty(); - let size = context.state_seq_get_size_ext(0, &flags); - let mut buffer = vec![0u8; size]; - let bytes_written = unsafe { context.state_seq_get_data_ext(&mut buffer, 0, &flags) }?; - - assert!(bytes_written > 0); - - let bytes_read = unsafe { context.state_seq_set_data_ext(&buffer, 0, &flags) }?; - - assert_eq!( - bytes_read, bytes_written, - "restoring the sequence state must consume exactly the bytes the snapshot produced" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_refilled_candidate_array_matches_a_freshly_built_one( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let mut context = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let prompt_tokens = model.str_to_token("Hello", AddBos::Never, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - - batch.add_sequence(&prompt_tokens, 0, false)?; - context.decode(&batch)?; - - let last_index = batch.n_tokens() - 1; - let mut reused_candidates = LlamaTokenDataArray::new( - vec![LlamaTokenData::new(LlamaToken::new(7), 1.0, 0.5)], - true, - ); - reused_candidates.selected = Some(0); - - context.fill_token_data_array_ith(last_index, &mut reused_candidates)?; - - assert_eq!(reused_candidates, context.token_data_array_ith(last_index)?); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 64, -)] -fn decoding_more_tokens_than_the_micro_batch_with_non_causal_attention_returns_an_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_attention_type(LlamaAttentionType::NonCausal), - )?; - let tokens = fixture.model.str_to_token( - &"hello ".repeat(100), - AddBos::Always, - ParseSpecialTokens::Always, - )?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - let n_tokens = batch.n_tokens(); - - assert_eq!( - context.decode(&batch), - Err(DecodeError::BatchExceedsMicroBatch { - n_tokens, - n_ubatch: context.n_ubatch(), - }) - ); - - Ok(()) -} diff --git a/llama-cpp-bindings-tests/tests/context_synchronization.rs b/llama-cpp-bindings-tests/tests/context_synchronization.rs new file mode 100644 index 000000000..c12dab152 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/context_synchronization.rs @@ -0,0 +1,24 @@ +use anyhow::Result; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn synchronize_completes_without_panic(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + context.synchronize(); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/debug_formatting.rs b/llama-cpp-bindings-tests/tests/debug_formatting.rs new file mode 100644 index 000000000..a32fc2f6f --- /dev/null +++ b/llama-cpp-bindings-tests/tests/debug_formatting.rs @@ -0,0 +1,40 @@ +use anyhow::Result; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn context_debug_format_contains_struct_name(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let debug_output = format!("{context:?}"); + + assert!(debug_output.contains("LlamaContext")); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128 +)] +fn model_debug_format_includes_struct_name_and_model_field(fixture: &LlamaFixture<'_>) { + let formatted = format!("{:?}", fixture.model); + + assert!(formatted.contains("LlamaModel")); + assert!(formatted.contains("model")); +} diff --git a/llama-cpp-bindings-tests/tests/decode_abort_control.rs b/llama-cpp-bindings-tests/tests/decode_abort_control.rs new file mode 100644 index 000000000..228a6412d --- /dev/null +++ b/llama-cpp-bindings-tests/tests/decode_abort_control.rs @@ -0,0 +1,103 @@ +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + +use anyhow::Result; +use llama_cpp_bindings::DecodeError; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn set_abort_flag_aborts_decode(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let abort_flag = Arc::new(AtomicBool::new(true)); + context.set_abort_flag(abort_flag); + + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + + let result = context.decode(&batch); + + assert_eq!(result, Err(DecodeError::Aborted)); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn set_abort_flag_false_allows_decode(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let abort_flag = Arc::new(AtomicBool::new(false)); + context.set_abort_flag(abort_flag); + + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + + let result = context.decode(&batch); + + assert!(result.is_ok()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn clear_abort_callback_allows_decode_with_flag_true(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let abort_flag = Arc::new(AtomicBool::new(true)); + context.set_abort_flag(abort_flag); + context.clear_abort_callback(); + + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + + let result = context.decode(&batch); + + assert!(result.is_ok()); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs b/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs new file mode 100644 index 000000000..7b14c8328 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs @@ -0,0 +1,104 @@ +use anyhow::Result; +use llama_cpp_bindings::DecodeError; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::context::params::LlamaAttentionType; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn decode_empty_batch_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let batch = LlamaBatch::new(ContextLimits::of(&context)); + + let result = context.decode(&batch); + + assert!(result.is_err()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 64, +)] +fn decoding_more_tokens_than_the_micro_batch_with_non_causal_attention_returns_an_error( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params) + .into_llama_context_params() + .with_attention_type(LlamaAttentionType::NonCausal), + )?; + let tokens = fixture.model.str_to_token( + &"hello ".repeat(100), + AddBos::Always, + ParseSpecialTokens::Always, + )?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + let n_tokens = batch.n_tokens(); + + assert_eq!( + context.decode(&batch), + Err(DecodeError::BatchExceedsMicroBatch { + n_tokens, + n_ubatch: context.n_ubatch(), + }) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 64, + embeddings = true, +)] +fn decoding_more_tokens_than_the_micro_batch_on_an_encoder_returns_an_error( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + let tokens = fixture.model.str_to_token( + &"hello ".repeat(100), + AddBos::Always, + ParseSpecialTokens::Always, + )?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + let n_tokens = batch.n_tokens(); + + assert_eq!( + context.decode(&batch), + Err(DecodeError::BatchExceedsMicroBatch { + n_tokens, + n_ubatch: context.n_ubatch(), + }) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/detokenization.rs b/llama-cpp-bindings-tests/tests/detokenization.rs new file mode 100644 index 000000000..e7d1597e0 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/detokenization.rs @@ -0,0 +1,326 @@ +use std::num::NonZeroU16; + +use anyhow::Result; +use llama_cpp_bindings::SampledToken; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_to_piece_bytes_returns_bytes_for_known_token(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let tokens = model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; + let bytes = model.token_to_piece_bytes(tokens[0], 32, false, None)?; + + assert_ne!(bytes, Vec::::new()); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_to_piece_handles_large_token_requiring_buffer_resize(fixture: &LlamaFixture<'_>) { + let model = fixture.model; + let mut decoder = encoding_rs::UTF_8.new_decoder(); + + for (token, _) in model.tokens(true).take(200) { + let result = model.token_to_piece(&SampledToken::Content(token), &mut decoder, true, None); + assert!(result.is_ok()); + } +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_to_piece_bytes_insufficient_buffer_returns_error( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let tokens = model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; + let result = model.token_to_piece_bytes(tokens[0], 1, false, None); + + assert!( + result + .unwrap_err() + .to_string() + .contains("Insufficient Buffer Space") + ); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_to_piece_with_lstrip_succeeds(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let mut decoder = encoding_rs::UTF_8.new_decoder(); + let tokens = model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; + let result = model.token_to_piece( + &SampledToken::Content(tokens[0]), + &mut decoder, + false, + NonZeroU16::new(1), + ); + + assert!(result.is_ok()); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_to_piece_decodes_reasoning_variant(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let mut decoder = encoding_rs::UTF_8.new_decoder(); + let tokens = model.str_to_token("hi", AddBos::Never, ParseSpecialTokens::Always)?; + + let piece = model.token_to_piece( + &SampledToken::Reasoning(tokens[0]), + &mut decoder, + true, + None, + )?; + + assert_ne!(piece, ""); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_to_piece_decodes_tool_call_variant(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let mut decoder = encoding_rs::UTF_8.new_decoder(); + let tokens = model.str_to_token("hi", AddBos::Never, ParseSpecialTokens::Always)?; + + let piece = + model.token_to_piece(&SampledToken::ToolCall(tokens[0]), &mut decoder, true, None)?; + + assert_ne!(piece, ""); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_to_piece_decodes_undeterminable_variant(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let mut decoder = encoding_rs::UTF_8.new_decoder(); + let tokens = model.str_to_token("hi", AddBos::Never, ParseSpecialTokens::Always)?; + + let piece = model.token_to_piece( + &SampledToken::Undeterminable(tokens[0]), + &mut decoder, + true, + None, + )?; + + assert_ne!(piece, ""); + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/dry_sampler.rs b/llama-cpp-bindings-tests/tests/dry_sampler.rs new file mode 100644 index 000000000..224e57f19 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/dry_sampler.rs @@ -0,0 +1,55 @@ +use std::ffi::CString; + +use llama_cpp_bindings::SamplingError; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn building_a_dry_sampler_with_sequence_breakers_does_not_abort(fixture: &LlamaFixture<'_>) { + let breakers: Vec<&[u8]> = vec![b"\n", b"\t"]; + let _sampler = LlamaSampler::dry(fixture.model, 1.5, 2.0, 128, 2, &breakers); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn dry_sampler_with_null_byte_in_seq_breakers_returns_error(fixture: &LlamaFixture<'_>) { + let breakers: Vec<&[u8]> = vec![b"hello\0world"]; + let result = LlamaSampler::dry(fixture.model, 1.5, 2.0, 128, 2, breakers); + + assert_eq!( + result.err(), + Some(SamplingError::DrySequenceBreakerContainsNul { + index: 0, + source: CString::new(&b"hello\0world"[..]).unwrap_err(), + }) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn building_a_dry_sampler_with_a_single_sequence_breaker_does_not_abort( + fixture: &LlamaFixture<'_>, +) { + let breakers: Vec<&[u8]> = vec![b"\n"]; + let _sampler = LlamaSampler::dry(fixture.model, 1.5, 2.0, 128, 2, &breakers); +} diff --git a/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs b/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs new file mode 100644 index 000000000..19376373a --- /dev/null +++ b/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs @@ -0,0 +1,85 @@ +use anyhow::Result; +use llama_cpp_bindings::ChatOutputCompleteness; +use llama_cpp_bindings_tests::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; +use llama_cpp_bindings_tests::paris_weather_tool_call::paris_weather_tool_call; +use llama_cpp_bindings_tests::recognized_answer::recognized_answer; +use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +fn assert_duck_types_paris_weather_call(fixture: &LlamaFixture<'_>, answer: &str) -> Result<()> { + let session = user_turn_chat_session( + fixture.model, + "What is the weather in Paris?", + GET_WEATHER_TOOLS_JSON, + false, + )?; + + assert!(!session.supports_tool_calls()); + assert_eq!( + recognized_answer(&session, answer, ChatOutputCompleteness::Finished)?.tool_calls, + [paris_weather_tool_call()] + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn deepseek_r1_8b_duck_types_gemma_paired_quote(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_duck_types_paris_weather_call( + fixture, + "<|tool_call>call:get_weather{location:<|\"|>Paris<|\"|>}", + ) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn deepseek_r1_8b_duck_types_glm_key_value_tags(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_duck_types_paris_weather_call( + fixture, + "get_weatherlocationParis", + ) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn deepseek_r1_8b_duck_types_mistral_bracketed_json(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_duck_types_paris_weather_call( + fixture, + r#"[TOOL_CALLS]get_weather[ARGS]{"location":"Paris"}"#, + ) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn deepseek_r1_8b_duck_types_qwen_xml(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_duck_types_paris_weather_call( + fixture, + "\n\n\nParis\n\n\n", + ) +} diff --git a/llama-cpp-bindings-tests/tests/embedding_generation.rs b/llama-cpp-bindings-tests/tests/embedding_generation.rs new file mode 100644 index 000000000..9f3846978 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/embedding_generation.rs @@ -0,0 +1,334 @@ +use std::time::Duration; + +use anyhow::Context; +use anyhow::Result; +use llama_cpp_bindings::SampledTokenClassifier; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::context::params::LlamaPoolingType; +use llama_cpp_bindings::ggml_time_us; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings_tests::normalize::normalize; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + n_threads_batch = 8, + embeddings = true, +)] +fn embedding_generation_produces_vectors(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + + let mut ctx = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + ) + .with_context(|| "unable to create context")?; + + let prompt = "Hello my name is"; + let tokens = model + .str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always) + .with_context(|| format!("failed to tokenize {prompt}"))?; + let prompt_token_count = u64::try_from(tokens.len())?; + + let n_ctx = usize::try_from(ctx.n_ctx())?; + assert!(tokens.len() <= n_ctx, "prompt exceeds context window size"); + + let t_main_start = ggml_time_us(); + + let mut classifier = SampledTokenClassifier::undeterminable(model); + let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); + classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; + + assert_eq!(classifier.pending_prompt_tokens(), prompt_token_count); + assert_eq!(classifier.usage().prompt_tokens, 0); + + ctx.clear_kv_cache(); + ctx.decode(&batch) + .with_context(|| "llama_decode() failed")?; + + let promoted = classifier.commit_prompt_tokens(); + assert_eq!(promoted, prompt_token_count); + + let embedding = ctx + .embeddings_seq_ith(0) + .with_context(|| "failed to get embeddings")?; + let normalized = normalize(embedding); + + let t_main_end = ggml_time_us(); + let duration = Duration::from_micros(u64::try_from(t_main_end - t_main_start)?); + + eprintln!( + "created embedding with {} dimensions in {:.2} s", + normalized.len(), + duration.as_secs_f32() + ); + + assert!( + !normalized.is_empty(), + "embedding should have at least one dimension" + ); + + let magnitude: f32 = normalized + .iter() + .map(|value| value * value) + .sum::() + .sqrt(); + assert!( + (magnitude - 1.0).abs() < 0.01, + "normalized embedding magnitude should be approximately 1.0, got {magnitude}" + ); + + let usage = classifier.into_usage(); + assert_eq!(usage.prompt_tokens, prompt_token_count); + assert_eq!(usage.completion_tokens(), 0); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + embeddings = true, +)] +fn decoding_with_embeddings_enabled_succeeds(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + + let result = context.decode(&batch); + + assert!(result.is_ok()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + embeddings = true, +)] +fn embeddings_seq_ith_returns_valid_embeddings(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let embeddings = context.embeddings_seq_ith(0)?; + + assert_eq!(embeddings.len(), usize::try_from(fixture.model.n_embd())?); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + n_seq_max = 4, + embeddings = true, +)] +fn multi_sequence_embeddings_returns_one_embedding_per_sequence( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let inputs = [ + "alpha is here", + "beta runs fast", + "gamma waits", + "delta jumps", + ]; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + + for (sequence_index, text) in inputs.iter().enumerate() { + let tokens = + fixture + .model + .str_to_token(text, AddBos::Always, ParseSpecialTokens::Always)?; + let sequence_id = i32::try_from(sequence_index)?; + + batch.add_sequence(&tokens, sequence_id, true)?; + } + + context.decode(&batch)?; + + let n_embd = usize::try_from(fixture.model.n_embd())?; + let mut collected: Vec> = Vec::with_capacity(inputs.len()); + + for sequence_index in 0..inputs.len() { + let sequence_id = i32::try_from(sequence_index)?; + let embedding = context.embeddings_seq_ith(sequence_id)?; + + assert_eq!( + embedding.len(), + n_embd, + "sequence {sequence_index} embedding length mismatch" + ); + + collected.push(embedding.to_vec()); + } + + for (left_index, left) in collected.iter().enumerate() { + for (right_index, right) in collected.iter().enumerate().skip(left_index + 1) { + assert_ne!( + left, right, + "embedding for sequence {left_index} must differ from sequence {right_index}", + ); + } + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + n_seq_max = 4, + embeddings = true, +)] +fn embeddings_returns_distinct_values_when_reused_batch_has_extra_capacity( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let iterations = [ + [ + "This is the first document with enough content to contribute meaningfully to the batch size calculation", + "This is the second document that should be processed in a potentially different batch from the first", + ], + [ + "This is the third document adding more content to ensure the total exceeds the configured chunk limit", + "This is the fourth document which should demonstrate that batching distributes across agent requests", + ], + ]; + + let n_embd = usize::try_from(fixture.model.n_embd())?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + let mut collected: Vec> = Vec::new(); + + for iteration_inputs in iterations { + for (sequence_index, text) in iteration_inputs.iter().enumerate() { + let tokens = + fixture + .model + .str_to_token(text, AddBos::Always, ParseSpecialTokens::Always)?; + let sequence_id = i32::try_from(sequence_index)?; + + batch.add_sequence(&tokens, sequence_id, true)?; + } + + context.clear_kv_cache(); + context.decode(&batch)?; + + for sequence_index in 0..iteration_inputs.len() { + let sequence_id = i32::try_from(sequence_index)?; + let embedding = context.embeddings_seq_ith(sequence_id)?; + + assert_eq!( + embedding.len(), + n_embd, + "iteration sequence {sequence_index} embedding length mismatch" + ); + + collected.push(embedding.to_vec()); + } + + batch.clear(); + } + + assert_eq!( + collected.len(), + iterations.iter().flatten().count(), + "expected one embedding per input across every iteration" + ); + + for (left_index, left) in collected.iter().enumerate() { + for (right_index, right) in collected.iter().enumerate().skip(left_index + 1) { + assert_ne!( + left, right, + "embedding {left_index} must differ from embedding {right_index} across reused-batch iterations", + ); + } + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + embeddings = true, +)] +fn embeddings_ith_returns_valid_embeddings(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params) + .into_llama_context_params() + .with_pooling_type(LlamaPoolingType::None), + )?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let last_index = i32::try_from(tokens.len() - 1)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let embeddings = context.embeddings_ith(last_index)?; + + assert_eq!(embeddings.len(), usize::try_from(fixture.model.n_embd())?); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/embedding_models.rs b/llama-cpp-bindings-tests/tests/embedding_models.rs deleted file mode 100644 index 34125e3e2..000000000 --- a/llama-cpp-bindings-tests/tests/embedding_models.rs +++ /dev/null @@ -1,830 +0,0 @@ -use std::num::NonZeroU8; -use std::time::Duration; - -use anyhow::Context; -use anyhow::Result; -use anyhow::anyhow; -use anyhow::bail; -use llama_cpp_bindings::BareJsonToolCalls; -use llama_cpp_bindings::ClearKvCacheSeqError; -use llama_cpp_bindings::CopyKvCacheSeqError; -use llama_cpp_bindings::DecodeError; -use llama_cpp_bindings::EmbeddingsError; -use llama_cpp_bindings::EncodeError; -use llama_cpp_bindings::KvCacheSeqAddError; -use llama_cpp_bindings::KvCacheSeqDivError; -use llama_cpp_bindings::KvCacheSeqPosMaxError; -use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_bindings::context::context_limits::ContextLimits; -use llama_cpp_bindings::context::params::LlamaPoolingType; -use llama_cpp_bindings::ggml_time_us; -use llama_cpp_bindings::gguf_context::GgufContext; -use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::model::ParseSpecialTokens; -use llama_cpp_bindings::model::params::LlamaModelParams; -use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_bindings_tests::prime_kv_cache::prime_kv_cache; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; -use toktrie::INVALID_TOKEN; -use toktrie::TokenizerEnv as _; - -fn normalize(input: &[f32]) -> Vec { - let magnitude = input - .iter() - .fold(0.0, |accumulator, &value| value.mul_add(value, accumulator)) - .sqrt(); - - input.iter().map(|&value| value / magnitude).collect() -} - -fn cosine_similarity(vec_a: &[f32], vec_b: &[f32]) -> f32 { - vec_a - .iter() - .zip(vec_b.iter()) - .map(|(left, right)| left * right) - .sum::() -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - n_threads_batch = 8, - embeddings = true, -)] -fn embedding_generation_produces_vectors(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - - let mut ctx = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - ) - .with_context(|| "unable to create context")?; - - let prompt = "Hello my name is"; - let tokens = model - .str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always) - .with_context(|| format!("failed to tokenize {prompt}"))?; - let prompt_token_count = u64::try_from(tokens.len())?; - - let n_ctx = usize::try_from(ctx.n_ctx())?; - assert!(tokens.len() <= n_ctx, "prompt exceeds context window size"); - - let t_main_start = ggml_time_us(); - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); - classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; - - assert_eq!(classifier.pending_prompt_tokens(), prompt_token_count); - assert_eq!(classifier.usage().prompt_tokens, 0); - - ctx.clear_kv_cache(); - ctx.decode(&batch) - .with_context(|| "llama_decode() failed")?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let embedding = ctx - .embeddings_seq_ith(0) - .with_context(|| "failed to get embeddings")?; - let normalized = normalize(embedding); - - let t_main_end = ggml_time_us(); - let duration = Duration::from_micros(u64::try_from(t_main_end - t_main_start)?); - - eprintln!( - "created embedding with {} dimensions in {:.2} s", - normalized.len(), - duration.as_secs_f32() - ); - - assert!( - !normalized.is_empty(), - "embedding should have at least one dimension" - ); - - let magnitude: f32 = normalized - .iter() - .map(|value| value * value) - .sum::() - .sqrt(); - assert!( - (magnitude - 1.0).abs() < 0.01, - "normalized embedding magnitude should be approximately 1.0, got {magnitude}" - ); - - let usage = classifier.into_usage(); - assert_eq!(usage.prompt_tokens, prompt_token_count); - assert_eq!(usage.completion_tokens(), 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - n_seq_max = 2, - n_threads_batch = 8, - embeddings = true, -)] -fn reranking_produces_scores(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - - let query = "What is machine learning?"; - let documents = [ - "Machine learning is a subset of artificial intelligence.", - "The weather today is sunny and warm.", - ]; - - let document_count = documents.len(); - assert_eq!( - u32::try_from(document_count)?, - fixture.context_params.n_seq_max, - "attribute n_seq_max must match the document count this trial expects", - ); - - let mut ctx = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - ) - .with_context(|| "unable to create context")?; - - let prompt_lines: Vec = documents - .iter() - .map(|document| format!("{query}{document}")) - .collect(); - - let tokens_lines_list = prompt_lines - .iter() - .map(|line| model.str_to_token(line, AddBos::Always, ParseSpecialTokens::Always)) - .collect::, _>>() - .with_context(|| "failed to tokenize prompts")?; - - let n_ctx = usize::try_from(ctx.n_ctx())?; - - if tokens_lines_list.iter().any(|tokens| n_ctx < tokens.len()) { - bail!("one of the provided prompts exceeds the size of the context window"); - } - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); - let t_main_start = ggml_time_us(); - - for (sequence_index, tokens) in tokens_lines_list.iter().enumerate() { - classifier.feed_prompt_sequence_to_batch( - &mut batch, - tokens, - i32::try_from(sequence_index)?, - false, - )?; - } - - let total_tokens: usize = tokens_lines_list.iter().map(Vec::len).sum(); - let total_token_count = u64::try_from(total_tokens)?; - - assert_eq!(classifier.pending_prompt_tokens(), total_token_count); - assert_eq!(classifier.usage().prompt_tokens, 0); - - ctx.clear_kv_cache(); - ctx.decode(&batch) - .with_context(|| "llama_decode() failed")?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, total_token_count); - - let mut embeddings = Vec::with_capacity(document_count); - - for sequence_index in 0..document_count { - let raw_embedding = ctx - .embeddings_seq_ith(i32::try_from(sequence_index)?) - .with_context(|| "failed to get sequence embeddings")?; - embeddings.push(normalize(raw_embedding)); - } - - let t_main_end = ggml_time_us(); - let duration = Duration::from_micros(u64::try_from(t_main_end - t_main_start)?); - - let tokens_per_second = f64::from(u32::try_from(total_tokens)?) / duration.as_secs_f64(); - - eprintln!( - "created embeddings for {total_tokens} tokens in {:.2} s, speed {tokens_per_second:.2} t/s", - duration.as_secs_f64(), - ); - - assert_eq!( - embeddings.len(), - document_count, - "should produce one embedding per document" - ); - - for (index, embedding) in embeddings.iter().enumerate() { - assert!( - !embedding.is_empty(), - "embedding {index} should not be empty" - ); - } - - let similarity = cosine_similarity(&embeddings[0], &embeddings[1]); - eprintln!("cosine similarity between document embeddings: {similarity:.4}"); - - assert!( - similarity.is_finite(), - "cosine similarity should be a finite number" - ); - - let usage = classifier.into_usage(); - assert_eq!(usage.prompt_tokens, total_token_count); - assert_eq!(usage.completion_tokens(), 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - embeddings = true, -)] -fn decode_with_embeddings_enabled(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - - let result = context.decode(&batch); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - embeddings = true, -)] -fn embeddings_seq_ith_returns_valid_embeddings(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let embeddings = context.embeddings_seq_ith(0)?; - - assert_eq!(embeddings.len(), usize::try_from(fixture.model.n_embd())?); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - n_seq_max = 4, - embeddings = true, -)] -fn multi_sequence_embeddings_returns_one_embedding_per_sequence( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let inputs = [ - "alpha is here", - "beta runs fast", - "gamma waits", - "delta jumps", - ]; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - - for (sequence_index, text) in inputs.iter().enumerate() { - let tokens = - fixture - .model - .str_to_token(text, AddBos::Always, ParseSpecialTokens::Always)?; - let sequence_id = i32::try_from(sequence_index)?; - - batch.add_sequence(&tokens, sequence_id, true)?; - } - - context.decode(&batch)?; - - let n_embd = usize::try_from(fixture.model.n_embd())?; - let mut collected: Vec> = Vec::with_capacity(inputs.len()); - - for sequence_index in 0..inputs.len() { - let sequence_id = i32::try_from(sequence_index)?; - let embedding = context.embeddings_seq_ith(sequence_id)?; - - assert_eq!( - embedding.len(), - n_embd, - "sequence {sequence_index} embedding length mismatch" - ); - - collected.push(embedding.to_vec()); - } - - for (left_index, left) in collected.iter().enumerate() { - for (right_index, right) in collected.iter().enumerate().skip(left_index + 1) { - assert_ne!( - left, right, - "embedding for sequence {left_index} must differ from sequence {right_index}", - ); - } - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - n_seq_max = 4, - embeddings = true, -)] -fn embeddings_returns_distinct_values_when_reused_batch_has_extra_capacity( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let iterations = [ - [ - "This is the first document with enough content to contribute meaningfully to the batch size calculation", - "This is the second document that should be processed in a potentially different batch from the first", - ], - [ - "This is the third document adding more content to ensure the total exceeds the configured chunk limit", - "This is the fourth document which should demonstrate that batching distributes across agent requests", - ], - ]; - - let n_embd = usize::try_from(fixture.model.n_embd())?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - let mut collected: Vec> = Vec::new(); - - for iteration_inputs in iterations { - for (sequence_index, text) in iteration_inputs.iter().enumerate() { - let tokens = - fixture - .model - .str_to_token(text, AddBos::Always, ParseSpecialTokens::Always)?; - let sequence_id = i32::try_from(sequence_index)?; - - batch.add_sequence(&tokens, sequence_id, true)?; - } - - context.clear_kv_cache(); - context.decode(&batch)?; - - for sequence_index in 0..iteration_inputs.len() { - let sequence_id = i32::try_from(sequence_index)?; - let embedding = context.embeddings_seq_ith(sequence_id)?; - - assert_eq!( - embedding.len(), - n_embd, - "iteration sequence {sequence_index} embedding length mismatch" - ); - - collected.push(embedding.to_vec()); - } - - batch.clear(); - } - - assert_eq!( - collected.len(), - iterations.iter().flatten().count(), - "expected one embedding per input across every iteration" - ); - - for (left_index, left) in collected.iter().enumerate() { - for (right_index, right) in collected.iter().enumerate().skip(left_index + 1) { - assert_ne!( - left, right, - "embedding {left_index} must differ from embedding {right_index} across reused-batch iterations", - ); - } - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - embeddings = true, -)] -fn embeddings_ith_returns_valid_embeddings(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_pooling_type(LlamaPoolingType::None), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let last_index = i32::try_from(tokens.len() - 1)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let embeddings = context.embeddings_ith(last_index)?; - - assert_eq!(embeddings.len(), usize::try_from(fixture.model.n_embd())?); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - embeddings = true, -)] -fn embeddings_ith_returns_null_embedding_error_for_non_embedding_token( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_pooling_type(LlamaPoolingType::None), - )?; - - assert_eq!( - context.embeddings_ith(999), - Err(EmbeddingsError::LogitsNotEnabled) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - embeddings = true, -)] -fn encode_succeeds_with_encoder_model(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - - let result = context.encode(&batch); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_add_succeeds_on_embedding_model(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - prime_kv_cache(fixture, &mut context)?; - - let result = context.kv_cache_seq_add(0, Some(0), None, 1); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_div_succeeds_on_embedding_model(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - prime_kv_cache(fixture, &mut context)?; - - let divisor = NonZeroU8::new(2).ok_or_else(|| anyhow!("2 is non-zero"))?; - let result = context.kv_cache_seq_div(0, Some(0), None, divisor); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128 -)] -fn embedding_model_exposes_tool_call_markers(fixture: &LlamaFixture<'_>) -> Result<()> { - let markers = fixture - .model - .tool_call_markers()? - .expect("Qwen3 embedding model must expose tool-call markers"); - - assert_ne!(markers.open, ""); - assert_ne!(markers.close, ""); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128 -)] -fn embedding_model_exposes_streaming_markers(fixture: &LlamaFixture<'_>) -> Result<()> { - let markers = fixture.model.streaming_markers()?; - - assert!(!markers.is_empty()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, - embeddings = true, -)] -fn an_encoder_context_keeps_no_memory(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(!fixture.build_context()?.has_memory()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, - embeddings = true, -)] -fn a_causal_embedding_context_keeps_memory(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(fixture.build_context()?.has_memory()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - n_threads_batch = 8, - embeddings = true, -)] -fn kv_cache_operations_respect_an_embedding_context_without_memory( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = fixture.build_context()?; - - context.clear_kv_cache(); - assert_eq!(context.clear_kv_cache_seq(Some(0), None, None), Ok(())); - context.kv_cache_seq_keep(0); - - assert_eq!( - context.copy_kv_cache_seq(0, 1, None, None), - Err(CopyKvCacheSeqError::MemoryHandleUnavailable) - ); - assert_eq!( - context.kv_cache_seq_add(0, None, None, 1), - Err(KvCacheSeqAddError::MemoryHandleUnavailable) - ); - let divisor = NonZeroU8::new(2).ok_or_else(|| anyhow!("2 is non-zero"))?; - assert_eq!( - context.kv_cache_seq_div(0, None, None, divisor), - Err(KvCacheSeqDivError::MemoryHandleUnavailable) - ); - assert_eq!( - context.kv_cache_seq_pos_max(0), - Err(KvCacheSeqPosMaxError::MemoryHandleUnavailable) - ); - assert!(matches!( - context.clear_kv_cache_seq(Some(u32::MAX), None, None), - Err(ClearKvCacheSeqError::SeqIdTooLarge(_)) - )); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 64, - embeddings = true, -)] -fn approximate_tok_env_falls_back_to_eos_when_eot_unavailable( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let LlamaToken(eos) = fixture - .model - .token_eos() - .context("model declares an EOS token")?; - - assert_eq!( - fixture.model.approximate_tok_env()?.tok_trie().eos_token(), - eos.cast_unsigned() - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 64, - embeddings = true, -)] -fn approximate_tok_env_of_a_model_without_eot_and_eos_has_no_eos( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model_without_eos_path = std::env::temp_dir().join(format!( - "llama_cpp_bindings_{}_model_without_eos.gguf", - std::process::id() - )); - let mut gguf_context = GgufContext::from_file(fixture.model_path)?; - - gguf_context.remove_key("tokenizer.ggml.eos_token_id")?; - gguf_context.write_to_file(&model_without_eos_path)?; - - let model_without_eos = LlamaModel::load_from_file( - fixture.backend, - &model_without_eos_path, - &LlamaModelParams::default(), - )?; - - std::fs::remove_file(&model_without_eos_path)?; - - assert_eq!( - model_without_eos - .approximate_tok_env()? - .tok_trie() - .eos_token(), - INVALID_TOKEN - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 64, - embeddings = true, -)] -fn decoding_more_tokens_than_the_micro_batch_on_an_encoder_returns_an_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = fixture.build_context()?; - let tokens = fixture.model.str_to_token( - &"hello ".repeat(100), - AddBos::Always, - ParseSpecialTokens::Always, - )?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - let n_tokens = batch.n_tokens(); - - assert_eq!( - context.decode(&batch), - Err(DecodeError::BatchExceedsMicroBatch { - n_tokens, - n_ubatch: context.n_ubatch(), - }) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 64, - embeddings = true, -)] -fn encoding_more_tokens_than_the_micro_batch_returns_an_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = fixture.build_context()?; - let tokens = fixture.model.str_to_token( - &"hello ".repeat(100), - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - let n_tokens = batch.n_tokens(); - - assert_eq!( - context.encode(&batch), - Err(EncodeError::BatchExceedsMicroBatch { - n_tokens, - n_ubatch: context.n_ubatch(), - }) - ); - - Ok(()) -} diff --git a/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs b/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs new file mode 100644 index 000000000..d4408d167 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs @@ -0,0 +1,119 @@ +use anyhow::Result; +use llama_cpp_bindings::EmbeddingsError; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::context::params::LlamaPoolingType; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn embeddings_ith_returns_error_when_embeddings_disabled(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + assert_eq!(context.embeddings_ith(0), Err(EmbeddingsError::NotEnabled)); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn embeddings_seq_ith_returns_error_when_embeddings_disabled( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + assert_eq!( + context.embeddings_seq_ith(0), + Err(EmbeddingsError::NotEnabled) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + embeddings = true, +)] +fn embeddings_seq_ith_returns_null_embedding_error_for_invalid_seq( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params) + .into_llama_context_params() + .with_pooling_type(LlamaPoolingType::Last), + )?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + assert_eq!( + context.embeddings_seq_ith(999), + Err(EmbeddingsError::SequenceEmbeddingUnavailable { + sequence_index: 999 + }) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + embeddings = true, +)] +fn embeddings_ith_returns_null_embedding_error_for_non_embedding_token( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params) + .into_llama_context_params() + .with_pooling_type(LlamaPoolingType::None), + )?; + + assert_eq!( + context.embeddings_ith(999), + Err(EmbeddingsError::LogitsNotEnabled) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs b/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs new file mode 100644 index 000000000..c9633a82c --- /dev/null +++ b/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs @@ -0,0 +1,41 @@ +use anyhow::Result; +use llama_cpp_bindings::EncodeError; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 64, + embeddings = true, +)] +fn encoding_more_tokens_than_the_micro_batch_returns_an_error( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + let tokens = fixture.model.str_to_token( + &"hello ".repeat(100), + AddBos::Never, + ParseSpecialTokens::Always, + )?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + let n_tokens = batch.n_tokens(); + + assert_eq!( + context.encode(&batch), + Err(EncodeError::BatchExceedsMicroBatch { + n_tokens, + n_ubatch: context.n_ubatch(), + }) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/encoder_requirement.rs b/llama-cpp-bindings-tests/tests/encoder_requirement.rs new file mode 100644 index 000000000..db62a094c --- /dev/null +++ b/llama-cpp-bindings-tests/tests/encoder_requirement.rs @@ -0,0 +1,63 @@ +use anyhow::Result; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn encode_on_non_encoder_model_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + + let result = context.encode(&batch); + + assert!(result.is_err()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + embeddings = true, +)] +fn encode_succeeds_with_encoder_model(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + + let result = context.encode(&batch); + + assert!(result.is_ok()); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs b/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs new file mode 100644 index 000000000..23e1f63c3 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs @@ -0,0 +1,122 @@ +use anyhow::Context; +use anyhow::Result; +use llama_cpp_bindings::SampledToken; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn is_eog_token_classifies_reasoning_variant(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let eos = model.token_eos().context("model declares an EOS token")?; + assert!(model.is_eog_token(&SampledToken::Reasoning(eos))); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn is_eog_token_classifies_tool_call_variant(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let eos = model.token_eos().context("model declares an EOS token")?; + assert!(model.is_eog_token(&SampledToken::ToolCall(eos))); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn is_eog_token_classifies_undeterminable_variant(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let eos = model.token_eos().context("model declares an EOS token")?; + assert!(model.is_eog_token(&SampledToken::Undeterminable(eos))); + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/generation_control.rs b/llama-cpp-bindings-tests/tests/generation_control.rs deleted file mode 100644 index 84133d889..000000000 --- a/llama-cpp-bindings-tests/tests/generation_control.rs +++ /dev/null @@ -1,2427 +0,0 @@ -use std::ffi::CStr; -use std::ffi::CString; -use std::io::Write; -use std::time::Duration; - -use anyhow::Context as _; -use anyhow::Result; -use anyhow::anyhow; -use anyhow::bail; -use llama_cpp_bindings::BareJsonToolCalls; -use llama_cpp_bindings::GbnfValidationError; -use llama_cpp_bindings::GenerationProgress; -use llama_cpp_bindings::GrammarError; -use llama_cpp_bindings::SampledToken; -use llama_cpp_bindings::SamplerAcceptError; -use llama_cpp_bindings::SamplingError; -use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_bindings::context::context_limits::ContextLimits; -use llama_cpp_bindings::ggml_time_us; -use llama_cpp_bindings::json_schema_to_grammar; -use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::llguidance_sampler::create_llg_sampler; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::LlamaChatMessage; -use llama_cpp_bindings::model::ParseSpecialTokens; -use llama_cpp_bindings::sampled_token_section::SampledTokenSection; -use llama_cpp_bindings::sampling::LlamaSampler; -use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn sample_returns_result_and_succeeds_with_valid_index(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut context = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let tokens = model.str_to_token("Hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - - batch.add_sequence(&tokens, 0, false)?; - - context.decode(&batch)?; - - let mut sampler = - LlamaSampler::chain_simple([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; - - let result = sampler.sample(&context, batch.n_tokens() - 1); - - assert!(result.is_ok()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn grammar_sampler_constrains_output_to_yes_or_no(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut context = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let prompt = "<|im_start|>user\nIs the sky blue? Answer yes or no.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; - let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - - batch.add_sequence(&tokens, 0, false)?; - - context.decode(&batch)?; - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::grammar(model, r"root ::= [Yy] [Ee] [Ss] | [Nn] [Oo]", "root")?, - LlamaSampler::temp(0.8)?, - LlamaSampler::greedy()?, - ])?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut outcomes = Vec::new(); - let turn = classifier.sample(&mut sampler, &context, batch.n_tokens() - 1, &mut outcomes)?; - classifier.finish(&mut outcomes); - - assert_eq!( - outcomes.len(), - 1, - "expected one finalised outcome after finishing" - ); - let outcome = &outcomes[0]; - - let raw_as_sampled = SampledToken::Content(turn.token); - assert!( - !model.is_eog_token(&raw_as_sampled), - "Grammar sampler should not allow EOS as first token" - ); - - let piece = outcome.piece.raw(); - let first_char = piece - .chars() - .next() - .ok_or_else(|| anyhow!("piece should have at least one character"))? - .to_lowercase() - .next() - .ok_or_else(|| anyhow!("lowercase iterator should yield a character"))?; - - assert!( - first_char == 'y' || first_char == 'n', - "Grammar should constrain first token to start with y/n, got: '{piece}'" - ); - assert_eq!( - classifier.usage().completion_tokens(), - 1, - "exactly one completion token sampled" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn json_schema_grammar_sampler_constrains_output_to_json(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut context = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let prompt = "<|im_start|>user\nWhat is 2+2? Respond with a JSON object.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; - let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - - batch.add_sequence(&tokens, 0, false)?; - - context.decode(&batch)?; - - let grammar_str = json_schema_to_grammar( - r#"{"type": "object", "properties": {"answer": {"type": "string"}}, "required": ["answer"]}"#, - )?; - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::grammar(model, &grammar_str, "root")?, - LlamaSampler::temp(0.8)?, - LlamaSampler::greedy()?, - ])?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut outcomes = Vec::new(); - let turn = classifier.sample(&mut sampler, &context, batch.n_tokens() - 1, &mut outcomes)?; - classifier.finish(&mut outcomes); - - assert_eq!( - outcomes.len(), - 1, - "expected one finalised outcome after finishing" - ); - let outcome = &outcomes[0]; - - let raw_as_sampled = SampledToken::Content(turn.token); - assert!( - !model.is_eog_token(&raw_as_sampled), - "Grammar sampler should not allow EOS as first token" - ); - - let piece = outcome.piece.raw(); - - assert!( - piece.starts_with('{'), - "JSON schema grammar should constrain first token to start with '{{', got: '{piece}'" - ); - assert_eq!( - classifier.usage().completion_tokens(), - 1, - "exactly one completion token sampled" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn sample_with_grammar_produces_constrained_output_in_loop( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let mut context = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let prompt = "<|im_start|>user\nIs the sky blue? yes or no<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; - let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; - - context.decode(&batch)?; - classifier.commit_prompt_tokens(); - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::grammar(model, r#"root ::= "yes" | "no""#, "root")?, - LlamaSampler::temp(0.8)?, - LlamaSampler::greedy()?, - ])?; - - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: 10, - } - .run()?; - - let lowercase = outcome.generated_raw.to_lowercase(); - assert!( - lowercase == "yes" || lowercase == "no", - "Grammar loop should produce 'yes' or 'no', got: '{}'", - outcome.generated_raw - ); - assert!( - outcome.eog_seen, - "loop must terminate via EOG once grammar accepts, not by exhausting the budget; outcome={outcome:?}" - ); - assert_eq!(outcome.observed_reasoning, 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(outcome.observed_tool_call, 0); - assert!(outcome.observed_content > 0); - - let usage = classifier.into_usage(); - assert_eq!(usage.completion_tokens(), outcome.observed_content); - assert_eq!(usage.reasoning_tokens, 0); - assert_eq!(usage.undeterminable_tokens, 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn sample_without_grammar_produces_multiple_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut context = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let prompt = - "<|im_start|>user\nSay hello<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; - let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - - batch.add_sequence(&tokens, 0, false)?; - - context.decode(&batch)?; - - let mut sampler = - LlamaSampler::chain_simple([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut outcomes = Vec::new(); - let mut sampled_count: u64 = 0; - - for (position, _) in (batch.n_tokens()..).zip(0..5) { - let turn = classifier.sample(&mut sampler, &context, -1, &mut outcomes)?; - - if turn.progress == GenerationProgress::Ended { - break; - } - - sampled_count += 1; - - let raw_as_sampled = SampledToken::Content(turn.token); - - batch.clear(); - batch.add(&raw_as_sampled, position, &[0], true)?; - - context.decode(&batch)?; - } - - classifier.finish(&mut outcomes); - - assert!( - sampled_count > 0, - "Should produce at least one token without grammar" - ); - let usage = classifier.into_usage(); - assert!( - usage.completion_tokens() >= sampled_count, - "completion_tokens ({}) must include the {sampled_count} non-EOG samples", - usage.completion_tokens() - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn dry_sampler_with_model(fixture: &LlamaFixture<'_>) -> Result<()> { - let breakers: Vec<&[u8]> = vec![b"\n", b"\t"]; - let _sampler = LlamaSampler::dry(fixture.model, 1.5, 2.0, 128, 2, &breakers); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn dry_sampler_with_null_byte_in_seq_breakers_returns_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let breakers: Vec<&[u8]> = vec![b"hello\0world"]; - let result = LlamaSampler::dry(fixture.model, 1.5, 2.0, 128, 2, breakers); - - assert_eq!( - result.err(), - Some(SamplingError::DrySequenceBreakerContainsNul { - index: 0, - source: CString::new(&b"hello\0world"[..]).unwrap_err(), - }) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn grammar_returns_sampler_for_valid_grammar(fixture: &LlamaFixture<'_>) -> Result<()> { - let sampler = LlamaSampler::grammar(fixture.model, "root ::= \"hello\"", "root"); - - assert!(sampler.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn grammar_lazy_returns_sampler_for_valid_grammar_with_patterns( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let patterns = vec!["\\{.*".to_owned()]; - let sampler = - LlamaSampler::grammar_lazy(fixture.model, "root ::= \"hello\"", "root", &patterns, &[]); - - assert!(sampler.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn grammar_lazy_with_root_not_found_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let patterns = vec!["function".to_owned()]; - let result = - LlamaSampler::grammar_lazy(fixture.model, "expr ::= \"hello\"", "root", &patterns, &[]); - - assert_eq!(result.err(), Some(GrammarError::RootNotFound)); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn grammar_lazy_with_null_byte_in_pattern_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let patterns = vec!["hel\0lo".to_owned()]; - let result = - LlamaSampler::grammar_lazy(fixture.model, "root ::= \"hello\"", "root", &patterns, &[]); - - assert_eq!( - result.err(), - Some(GrammarError::TriggerPatternContainsNul( - CString::new("hel\0lo").expect_err("the pattern carries a nul byte") - )) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn grammar_lazy_returns_sampler_for_valid_grammar_with_trigger_tokens( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let trigger_tokens = - fixture - .model - .str_to_token("{", AddBos::Never, ParseSpecialTokens::Always)?; - - assert!( - !trigger_tokens.is_empty(), - "the tokenizer must produce at least one trigger token" - ); - - let sampler = LlamaSampler::grammar_lazy( - fixture.model, - "root ::= \"hello\"", - "root", - &[], - &trigger_tokens, - ); - - assert!( - sampler.is_ok(), - "trigger tokens alone must be enough to build a lazy grammar sampler" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn grammar_lazy_with_null_byte_in_grammar_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = - LlamaSampler::grammar_lazy(fixture.model, "root ::= \"hel\0lo\"", "root", &[], &[]); - - assert_eq!( - result.err(), - Some(GrammarError::GrammarContainsNul( - CString::new("root ::= \"hel\0lo\"").expect_err("the grammar carries a nul byte") - )) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn grammar_lazy_with_malformed_regex_returns_invalid_trigger_pattern( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let patterns = vec!["[".to_owned()]; - let result = - LlamaSampler::grammar_lazy(fixture.model, "root ::= \"hello\"", "root", &patterns, &[]); - - assert!(matches!( - result, - Err(GrammarError::InvalidTriggerPattern { .. }), - )); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn llguidance_method_creates_sampler(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = LlamaSampler::llguidance(fixture.model, "regex", r"yes|no"); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn logit_bias_with_empty_biases_succeeds(_fixture: &LlamaFixture<'_>) -> Result<()> { - let result = LlamaSampler::logit_bias(0, &[]); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn dry_sampler_with_root_not_found_grammar_does_not_apply( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let breakers: Vec<&[u8]> = vec![b"\n"]; - let _sampler = LlamaSampler::dry(fixture.model, 1.5, 2.0, 128, 2, &breakers); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn accept_many_iterates_over_borrowed_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut sampler = LlamaSampler::chain_simple([LlamaSampler::greedy()?])?; - let tokens = vec![ - fixture - .model - .token_bos() - .context("model declares a BOS token")?, - fixture - .model - .token_eos() - .context("model declares an EOS token")?, - ]; - - sampler.accept_many(&tokens)?; - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn with_tokens_returns_self_after_accepting_each_token(fixture: &LlamaFixture<'_>) -> Result<()> { - let sampler = LlamaSampler::chain_simple([LlamaSampler::greedy()?])?; - let tokens = [ - fixture - .model - .token_bos() - .context("model declares a BOS token")?, - fixture - .model - .token_eos() - .context("model declares an EOS token")?, - ]; - - let _consumed = sampler.with_tokens(tokens.iter().copied())?; - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn accept_consumes_a_single_token(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut sampler = LlamaSampler::chain_simple([LlamaSampler::greedy()?])?; - - sampler.accept( - fixture - .model - .token_bos() - .context("model declares a BOS token")?, - )?; - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn accept_succeeds_for_a_valid_token(_fixture: &LlamaFixture<'_>) -> Result<()> { - let mut sampler = LlamaSampler::chain_simple([LlamaSampler::greedy()?])?; - - sampler.accept(LlamaToken::new(0))?; - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn apply_runs_sampler_over_token_data_array(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("Hi", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let mut data_array = context.token_data_array_ith(batch.n_tokens() - 1)?; - let sampler = LlamaSampler::greedy()?; - sampler.apply(&mut data_array)?; - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn sample_returns_token_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("Hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - let mut sampler = - LlamaSampler::chain_simple([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; - let result = sampler.sample(&context, batch.n_tokens() - 1); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn raw_prompt_completion_with_timing(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let backend = fixture.backend; - let mut ctx = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - ) - .with_context(|| "unable to create context")?; - - let prompt = "Hello my name is"; - let max_generated_tokens: i32 = 64; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let tokens_list = model - .str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always) - .with_context(|| format!("failed to tokenize {prompt}"))?; - let prompt_token_count = u64::try_from(tokens_list.len())?; - - let mut decoder = encoding_rs::UTF_8.new_decoder(); - - for token in &tokens_list { - eprint!( - "{}", - model.token_to_piece(&SampledToken::Content(*token), &mut decoder, true, None)? - ); - } - std::io::stderr().flush()?; - - let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); - classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens_list, 0, false)?; - - assert_eq!(classifier.pending_prompt_tokens(), prompt_token_count); - assert_eq!(classifier.usage().prompt_tokens, 0); - - ctx.decode(&batch) - .with_context(|| "llama_decode() failed")?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - assert_eq!(classifier.usage().prompt_tokens, prompt_token_count); - - let mut sampler = - LlamaSampler::chain_simple([LlamaSampler::dist(1234)?, LlamaSampler::greedy()?])?; - let initial_position = batch.n_tokens(); - let t_main_start = ggml_time_us(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut ctx, - batch: &mut batch, - initial_position, - max_generated_tokens, - } - .run()?; - let t_main_end = ggml_time_us(); - let duration = Duration::from_micros(u64::try_from(t_main_end - t_main_start)?); - let total_observed = - outcome.observed_content + outcome.observed_reasoning + outcome.observed_undeterminable; - - let tokens_per_second = f64::from(u32::try_from(total_observed)?) / duration.as_secs_f64(); - - eprintln!( - "\ndecoded {total_observed} tokens in {:.2} s, speed {tokens_per_second:.2} t/s", - duration.as_secs_f64(), - ); - - assert!( - !outcome.generated_raw.is_empty(), - "model should generate at least one token" - ); - assert_eq!( - outcome.observed_tool_call, 0, - "raw prompt without tool-call markers must not produce ToolCall tokens; \ - outcome={outcome:?}" - ); - assert!( - total_observed > 0, - "model must produce at least one classified token; outcome={outcome:?}" - ); - - let usage = classifier.into_usage(); - assert_eq!( - usage.prompt_tokens, prompt_token_count, - "prompt_tokens must equal the tokenizer's prompt length" - ); - assert_eq!( - usage.content_tokens, outcome.observed_content, - "content_tokens must equal observed Content variants" - ); - assert_eq!( - usage.reasoning_tokens, outcome.observed_reasoning, - "reasoning_tokens must equal observed Reasoning variants" - ); - assert_eq!( - usage.undeterminable_tokens, outcome.observed_undeterminable, - "undeterminable_tokens must equal observed Undeterminable variants" - ); - assert_eq!( - usage.tool_call_tokens, outcome.observed_tool_call, - "tool_call_tokens must equal observed ToolCall variants" - ); - assert_eq!( - usage.completion_tokens(), - total_observed, - "completion_tokens must equal Content + Reasoning + Undeterminable" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128, -)] -fn chat_inference_produces_coherent_output(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let backend = fixture.backend; - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let chat_template = model.chat_template(None)?; - let messages = vec![LlamaChatMessage::new( - "user".to_string(), - "Hello! How are you?".to_string(), - )?]; - let prompt = model.apply_chat_template(&chat_template, &messages, true, true)?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let tokens = model.str_to_token(&prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let prompt_token_count = u64::try_from(tokens.len())?; - - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; - - assert_eq!(classifier.pending_prompt_tokens(), prompt_token_count); - assert_eq!(classifier.usage().prompt_tokens, 0); - - context.decode(&batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::greedy()?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: 512, - } - .run()?; - - println!(); - - assert!( - !outcome.generated_raw.is_empty(), - "model should generate at least one token" - ); - let total_observed = - outcome.observed_content + outcome.observed_reasoning + outcome.observed_undeterminable; - assert!( - total_observed > 0, - "model must produce at least one classified token; outcome={outcome:?}" - ); - assert_eq!( - outcome.observed_tool_call, 0, - "chat without tool definitions must not produce ToolCall tokens; outcome={outcome:?}" - ); - - let usage = classifier.into_usage(); - - assert_eq!( - usage.prompt_tokens, prompt_token_count, - "prompt_tokens must equal the tokenizer's prompt length" - ); - assert_eq!( - usage.content_tokens, outcome.observed_content, - "content_tokens must equal observed Content variants" - ); - assert_eq!( - usage.reasoning_tokens, outcome.observed_reasoning, - "reasoning_tokens must equal observed Reasoning variants" - ); - assert_eq!( - usage.undeterminable_tokens, outcome.observed_undeterminable, - "undeterminable_tokens must equal observed Undeterminable variants" - ); - assert_eq!( - usage.completion_tokens(), - total_observed, - "completion_tokens must equal Content + Reasoning + Undeterminable" - ); - assert_eq!( - usage.tool_call_tokens, outcome.observed_tool_call, - "tool_call_tokens must equal observed ToolCall variants" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn json_schema_constrains_output(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let backend = fixture.backend; - - let prompt = "The weather in Paris is sunny and 22 degrees. Extract as JSON:\n"; - - let mut ctx = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let tokens_list = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; - - let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); - let last_index = i32::try_from(tokens_list.len())? - 1; - - for (index, token) in (0_i32..).zip(&tokens_list) { - batch.add( - &SampledToken::Content(*token), - index, - &[0], - index == last_index, - )?; - } - - ctx.decode(&batch)?; - - let schema = r#"{ - "type": "object", - "properties": { - "city": { "type": "string" }, - "temperature": { "type": "number" } - }, - "required": ["city", "temperature"] -}"#; - - let llg_sampler = LlamaSampler::llguidance(model, "json", schema)?; - let mut sampler = LlamaSampler::chain_simple([llg_sampler, LlamaSampler::greedy()?])?; - - let mut n_cur = batch.n_tokens(); - let mut decoder = encoding_rs::UTF_8.new_decoder(); - let mut generated = String::new(); - - while n_cur <= 128 { - let token = SampledToken::Content(sampler.sample(&ctx, batch.n_tokens() - 1)?); - - if model.is_eog_token(&token) { - break; - } - - let output_string = model.token_to_piece(&token, &mut decoder, true, None)?; - generated.push_str(&output_string); - print!("{output_string}"); - std::io::stdout().flush()?; - - batch.clear(); - batch.add(&token, n_cur, &[0], true)?; - n_cur += 1; - ctx.decode(&batch)?; - } - - println!(); - - let parsed = serde_json::Deserializer::from_str(&generated) - .into_iter::() - .next() - .ok_or_else(|| anyhow!("model produced no JSON value"))??; - - assert!(parsed.get("city").is_some()); - assert!(parsed.get("temperature").is_some()); - - Ok(()) -} - -const JSON_SCHEMA: &str = - r#"{"type":"object","properties":{"answer":{"type":"string"}},"required":["answer"]}"#; -const REGEX_GRAMMAR: &str = r"yes|no"; -const LARK_GRAMMAR: &str = r#"start: "yes" | "no""#; - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn creates_sampler_with_valid_json_schema(fixture: &LlamaFixture<'_>) -> Result<()> { - let sampler = create_llg_sampler(fixture.model, "json", JSON_SCHEMA)?; - - assert!(!sampler.as_ptr().is_null()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn creates_sampler_with_valid_regex_grammar(fixture: &LlamaFixture<'_>) -> Result<()> { - let sampler = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; - - assert!(!sampler.as_ptr().is_null()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn creates_sampler_with_valid_lark_grammar(fixture: &LlamaFixture<'_>) -> Result<()> { - let sampler = create_llg_sampler(fixture.model, "lark", LARK_GRAMMAR)?; - - assert!(!sampler.as_ptr().is_null()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn returns_error_for_unknown_grammar_kind(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = create_llg_sampler(fixture.model, "not_a_real_kind", "anything"); - - assert!(result.is_err()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn returns_error_for_malformed_json_schema(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = create_llg_sampler(fixture.model, "json", "{this is not valid json"); - - assert!(result.is_err()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn returns_error_for_malformed_regex(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = create_llg_sampler(fixture.model, "regex", "[invalid"); - - assert!(result.is_err()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn name_callback_returns_llguidance(fixture: &LlamaFixture<'_>) -> Result<()> { - let sampler = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; - - let name_ptr = unsafe { llama_cpp_bindings_sys::llama_sampler_name(sampler.as_ptr()) }; - assert!(!name_ptr.is_null()); - let name = unsafe { CStr::from_ptr(name_ptr) }.to_str()?; - - assert_eq!(name, "llguidance"); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn clone_via_ffi_creates_independent_sampler(fixture: &LlamaFixture<'_>) -> Result<()> { - let sampler = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; - - let cloned = unsafe { llama_cpp_bindings_sys::llama_sampler_clone(sampler.as_ptr()) }; - - assert!(!cloned.is_null()); - - unsafe { llama_cpp_bindings_sys::llama_sampler_free(cloned) }; - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn samples_token_constrained_by_grammar(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let backend = fixture.backend; - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let prompt = "Answer yes or no:"; - let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let llg_sampler = create_llg_sampler(model, "regex", REGEX_GRAMMAR)?; - let mut chain = LlamaSampler::chain_simple([llg_sampler, LlamaSampler::greedy()?])?; - - let token = chain.sample(&context, batch.n_tokens() - 1)?; - assert!( - token.0 >= 0, - "grammar-constrained sampling must yield a valid token id without the grammar rejecting it" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn reset_rolls_back_accepted_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut sampler = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; - - let yes_tokens = - fixture - .model - .str_to_token("yes", AddBos::Never, ParseSpecialTokens::Always)?; - let first_allowed_token = *yes_tokens - .first() - .ok_or_else(|| anyhow!("the tokenizer must produce a token for \"yes\""))?; - - sampler - .accept(first_allowed_token) - .context("the grammar must accept the first token of an allowed word")?; - - sampler - .reset() - .context("reset must roll a healthy grammar matcher back to its initial state")?; - - assert_eq!( - sampler.accept(first_allowed_token), - Ok(()), - "after reset the same token must be acceptable again, which is what rollback means" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn accept_maps_an_out_of_vocabulary_token_to_grammar_callback_failure( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut sampler = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; - - let huge_token = LlamaToken(i32::MAX - 1); - let result = sampler.accept(huge_token); - - let Err(SamplerAcceptError::GrammarCallbackFailed { message }) = result else { - panic!("expected a grammar callback failure, got {result:?}"); - }; - assert!(message.contains("token id 2147483646 out of range")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn approximate_tok_env_drives_consistent_grammar_constraint( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let first = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; - let second = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; - - assert!(!first.as_ptr().is_null()); - assert!(!second.as_ptr().is_null()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn llguidance_chain_samples_a_valid_token(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let backend = fixture.backend; - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let tokens = model.str_to_token("Answer:", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let llg_sampler = create_llg_sampler(model, "regex", REGEX_GRAMMAR)?; - let mut chain = LlamaSampler::chain_simple([llg_sampler, LlamaSampler::greedy()?])?; - let token = chain.sample(&context, batch.n_tokens() - 1)?; - - assert!(token.0 >= 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn classifier_starts_in_pending_section_for_default_fixture( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let classifier = fixture - .model - .sampled_token_classifier(BareJsonToolCalls::Detect)?; - - assert_eq!(classifier.current_section(), SampledTokenSection::Pending); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn classifier_construction_is_idempotent_across_calls(fixture: &LlamaFixture<'_>) -> Result<()> { - let first = fixture - .model - .sampled_token_classifier(BareJsonToolCalls::Detect)?; - let second = fixture - .model - .sampled_token_classifier(BareJsonToolCalls::Detect)?; - - assert_eq!(first.current_section(), second.current_section()); - assert_eq!(first.usage(), second.usage()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn finishing_releases_an_unmatched_token_with_its_visible_and_raw_piece( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let [ordinary_token] = - model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?[..] - else { - bail!("\"hello\" must be a single token"); - }; - - let mut outcomes = Vec::new(); - classifier.ingest(ordinary_token, &mut outcomes)?; - classifier.finish(&mut outcomes); - - assert_eq!(outcomes.len(), 1); - let outcome = &outcomes[0]; - assert!(matches!( - outcome.sampled_token, - SampledToken::Undeterminable(_) - )); - assert_eq!(outcome.piece.visible(), outcome.piece.raw()); - assert_eq!(classifier.usage().undeterminable_tokens, 1); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn ingest_counts_every_token_through_the_end_of_generation( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let [ordinary_token] = - model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?[..] - else { - bail!("\"hello\" must be a single token"); - }; - let mut outcomes = Vec::new(); - - classifier.ingest(ordinary_token, &mut outcomes)?; - - assert_eq!( - classifier.ingest( - model.token_eos().context("model declares an EOS token")?, - &mut outcomes - )?, - GenerationProgress::Ended - ); - - assert_eq!(classifier.usage().undeterminable_tokens, 2); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn ingest_unmatched_prompt_tokens_does_not_record_usage(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let usage_before = *classifier.usage(); - - classifier.ingest_prompt_token(model.token_bos().context("model declares a BOS token")?); - classifier.ingest_prompt_tokens(&[ - model.token_eos().context("model declares an EOS token")?, - model.token_nl().context("model declares a newline token")?, - ]); - - assert_eq!(*classifier.usage(), usage_before); - assert_eq!(classifier.current_section(), SampledTokenSection::Pending); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn feed_prompt_to_batch_increments_pending_prompt_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut batch = LlamaBatch::new(ContextLimits { - n_batch: 8, - n_ubatch: 8, - n_seq_max: 1, - n_vocab: model.n_vocab(), - }); - - classifier.feed_prompt_to_batch( - &mut batch, - model.token_bos().context("model declares a BOS token")?, - 0, - &[0], - false, - )?; - classifier.feed_prompt_to_batch( - &mut batch, - model.token_eos().context("model declares an EOS token")?, - 1, - &[0], - false, - )?; - - assert_eq!(classifier.pending_prompt_tokens(), 2); - assert_eq!(batch.n_tokens(), 2); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn feed_prompt_sequence_to_batch_stages_all_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut batch = LlamaBatch::new(ContextLimits { - n_batch: 8, - n_ubatch: 8, - n_seq_max: 1, - n_vocab: model.n_vocab(), - }); - - let tokens = vec![ - model.token_bos().context("model declares a BOS token")?, - model.token_eos().context("model declares an EOS token")?, - model.token_nl().context("model declares a newline token")?, - ]; - classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; - - assert_eq!(classifier.pending_prompt_tokens(), 3); - assert_eq!(batch.n_tokens(), 3); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn commit_prompt_tokens_promotes_pending_count_to_usage_and_clears( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut batch = LlamaBatch::new(ContextLimits { - n_batch: 8, - n_ubatch: 8, - n_seq_max: 1, - n_vocab: model.n_vocab(), - }); - - classifier.feed_prompt_to_batch( - &mut batch, - model.token_bos().context("model declares a BOS token")?, - 0, - &[0], - false, - )?; - classifier.feed_prompt_to_batch( - &mut batch, - model.token_eos().context("model declares an EOS token")?, - 1, - &[0], - false, - )?; - - let promoted = classifier.commit_prompt_tokens(); - - assert_eq!(promoted, 2); - assert_eq!(classifier.pending_prompt_tokens(), 0); - assert_eq!(classifier.usage().prompt_tokens, 2); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn discard_pending_prompt_tokens_clears_count_without_recording_usage( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut batch = LlamaBatch::new(ContextLimits { - n_batch: 8, - n_ubatch: 8, - n_seq_max: 1, - n_vocab: model.n_vocab(), - }); - - classifier.feed_prompt_to_batch( - &mut batch, - model.token_bos().context("model declares a BOS token")?, - 0, - &[0], - false, - )?; - - let discarded = classifier.discard_pending_prompt_tokens(); - - assert_eq!(discarded, 1); - assert_eq!(classifier.pending_prompt_tokens(), 0); - assert_eq!(classifier.usage().prompt_tokens, 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn diagnose_tool_call_synthetic_renders_applies_the_template_to_both_probes( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let renders = fixture.model.diagnose_tool_call_synthetic_renders()?; - - assert!( - !renders.without_tools.is_empty(), - "the probe render without tool calls must not be empty" - ); - assert!( - !renders.with_tools.is_empty(), - "the probe render with a tool call must not be empty" - ); - for render in [&renders.without_tools, &renders.with_tools] { - assert!( - render.contains("Please use the tool"), - "each probe render must contain the synthetic user turn, which is what proves the \ - template was applied; got: {render:?}" - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_grammar_with_a_token_reference_constrains_the_first_token( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let [think_token] = - model.str_to_token("", AddBos::Never, ParseSpecialTokens::Always)?[..] - else { - bail!(" must be a single token"); - }; - let mut context = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let prompt_tokens = model.str_to_token( - "<|im_start|>user\nSay hi<|im_end|>\n<|im_start|>assistant\n", - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - - batch.add_sequence(&prompt_tokens, 0, false)?; - context.decode(&batch)?; - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::grammar(model, r#"root ::= "x""#, "root")?, - LlamaSampler::greedy()?, - ])?; - - assert_eq!(sampler.sample(&context, batch.n_tokens() - 1)?, think_token); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_grammar_without_its_root_reports_the_missing_root( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - assert_eq!( - LlamaSampler::grammar(fixture.model, r#"answer ::= "x""#, "root").err(), - Some(GrammarError::RootNotFound) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_grammar_with_a_syntax_error_reports_it(fixture: &LlamaFixture<'_>) -> Result<()> { - assert_eq!( - LlamaSampler::grammar(fixture.model, "root ::= (", "root").err(), - Some(GrammarError::GrammarRejected( - GbnfValidationError::SyntaxError - )) - ); - - Ok(()) -} diff --git a/llama-cpp-bindings-tests/tests/generation_end_classification.rs b/llama-cpp-bindings-tests/tests/generation_end_classification.rs new file mode 100644 index 000000000..d355271e5 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/generation_end_classification.rs @@ -0,0 +1,109 @@ +use anyhow::Result; +use llama_cpp_bindings::BareJsonToolCalls; +use llama_cpp_bindings::SampledToken; +use llama_cpp_bindings::SampledTokenClassifier; +use llama_cpp_bindings::ingest_outcome::IngestOutcome; +use llama_cpp_bindings_tests::ingested_generation::ingested_generation; +use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +const END_OF_GENERATION_TOKENS: u64 = 1; + +fn visible_text(outcomes: &[IngestOutcome]) -> String { + outcomes + .iter() + .map(|outcome| outcome.piece.visible()) + .collect() +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_classifier_ends_generation_without_emitting_the_end_of_generation_token( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + const GENERATED_TEXT: &str = "The answer is four."; + + let session = user_turn_chat_session(fixture.model, "What is 2 + 2?", "[]", false)?; + let mut classifier = + SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + + assert_eq!( + visible_text(&ingested_generation( + &mut classifier, + fixture.model, + GENERATED_TEXT + )?), + GENERATED_TEXT + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn gemma3_releases_a_held_bare_json_prefix_as_content_when_generation_ends( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + const GENERATED_TEXT: &str = r#"{"answer": 4"#; + + let session = user_turn_chat_session(fixture.model, "What is 2 + 2?", "[]", false)?; + let mut classifier = + SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let outcomes = ingested_generation(&mut classifier, fixture.model, GENERATED_TEXT)?; + + assert!(!session.supports_tool_calls()); + assert_eq!(visible_text(&outcomes), GENERATED_TEXT); + assert!( + outcomes + .iter() + .all(|outcome| matches!(outcome.sampled_token, SampledToken::Content(_))), + "{outcomes:?}" + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn gemma3_attributes_a_bare_json_tool_call_object_to_the_tool_call( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + const GENERATED_TEXT: &str = r#"{"name": "get_weather", "arguments": {"location": "Paris"}}"#; + + let session = user_turn_chat_session(fixture.model, "What is 2 + 2?", "[]", false)?; + let mut classifier = + SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let outcomes = ingested_generation(&mut classifier, fixture.model, GENERATED_TEXT)?; + + assert!( + outcomes + .iter() + .all(|outcome| matches!(outcome.sampled_token, SampledToken::ToolCall(_))), + "{outcomes:?}" + ); + assert_eq!( + classifier.usage().tool_call_tokens, + u64::try_from(outcomes.len())? + END_OF_GENERATION_TOKENS + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/grammar_sampler.rs b/llama-cpp-bindings-tests/tests/grammar_sampler.rs new file mode 100644 index 000000000..279f3900a --- /dev/null +++ b/llama-cpp-bindings-tests/tests/grammar_sampler.rs @@ -0,0 +1,293 @@ +use anyhow::Result; +use anyhow::anyhow; +use anyhow::bail; +use llama_cpp_bindings::BareJsonToolCalls; +use llama_cpp_bindings::GbnfValidationError; +use llama_cpp_bindings::GrammarError; +use llama_cpp_bindings::SampledToken; +use llama_cpp_bindings::SampledTokenClassifier; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; +use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn grammar_sampler_constrains_output_to_yes_or_no(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let mut context = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let prompt = "<|im_start|>user\nIs the sky blue? Answer yes or no.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; + let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + + batch.add_sequence(&tokens, 0, false)?; + + context.decode(&batch)?; + + let mut sampler = LlamaSampler::chain_simple([ + LlamaSampler::grammar(model, r"root ::= [Yy] [Ee] [Ss] | [Nn] [Oo]", "root")?, + LlamaSampler::temp(0.8)?, + LlamaSampler::greedy()?, + ])?; + + let mut classifier = SampledTokenClassifier::undeterminable(model); + let mut outcomes = Vec::new(); + let turn = classifier.sample(&mut sampler, &context, batch.n_tokens() - 1, &mut outcomes)?; + classifier.finish(&mut outcomes)?; + + assert_eq!( + outcomes.len(), + 1, + "expected one finalised outcome after finishing" + ); + let outcome = &outcomes[0]; + + let raw_as_sampled = SampledToken::Content(turn.token); + assert!( + !model.is_eog_token(&raw_as_sampled), + "Grammar sampler should not allow EOS as first token" + ); + + let piece = outcome.piece.raw(); + let first_char = piece + .chars() + .next() + .ok_or_else(|| anyhow!("piece should have at least one character"))? + .to_lowercase() + .next() + .ok_or_else(|| anyhow!("lowercase iterator should yield a character"))?; + + assert!( + first_char == 'y' || first_char == 'n', + "Grammar should constrain first token to start with y/n, got: '{piece}'" + ); + assert_eq!( + classifier.usage().completion_tokens(), + 1, + "exactly one completion token sampled" + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn sample_with_grammar_produces_constrained_output_in_loop( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let mut context = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let session = closed_reasoning_chat_session(model, "Is the sky blue? yes or no", "[]")?; + let tokens = session.tokenize_prompt()?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + + let mut classifier = + SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; + + context.decode(&batch)?; + classifier.commit_prompt_tokens(); + + let mut sampler = LlamaSampler::chain_simple([ + LlamaSampler::grammar(model, r#"root ::= "yes" | "no""#, "root")?, + LlamaSampler::temp(0.8)?, + LlamaSampler::greedy()?, + ])?; + + let initial_position = batch.n_tokens(); + let outcome = ClassifySampleLoop { + classifier: &mut classifier, + sampler: &mut sampler, + context: &mut context, + batch: &mut batch, + initial_position, + max_generated_tokens: 10, + } + .run()?; + + let lowercase = outcome.generated.text().to_lowercase(); + assert!( + lowercase == "yes" || lowercase == "no", + "Grammar loop should produce 'yes' or 'no', got: '{}'", + outcome.generated.text() + ); + assert!( + outcome.eog_seen, + "loop must terminate via EOG once grammar accepts, not by exhausting the budget; outcome={outcome:?}" + ); + assert_eq!(outcome.observed_reasoning, 0); + assert_eq!(outcome.observed_undeterminable, 0); + assert_eq!(outcome.observed_tool_call, 0); + assert!(outcome.observed_content > 0); + + let usage = classifier.into_usage(); + assert_eq!(usage.completion_tokens(), outcome.observed_content); + assert_eq!(usage.reasoning_tokens, 0); + assert_eq!(usage.undeterminable_tokens, 0); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn grammar_returns_sampler_for_valid_grammar(fixture: &LlamaFixture<'_>) { + let sampler = LlamaSampler::grammar(fixture.model, "root ::= \"hello\"", "root"); + + assert!(sampler.is_ok()); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_grammar_with_a_token_reference_constrains_the_first_token( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let [think_token] = + model.str_to_token("", AddBos::Never, ParseSpecialTokens::Always)?[..] + else { + bail!(" must be a single token"); + }; + let mut context = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let prompt_tokens = model.str_to_token( + "<|im_start|>user\nSay hi<|im_end|>\n<|im_start|>assistant\n", + AddBos::Never, + ParseSpecialTokens::Always, + )?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + + batch.add_sequence(&prompt_tokens, 0, false)?; + context.decode(&batch)?; + + let mut sampler = LlamaSampler::chain_simple([ + LlamaSampler::grammar(model, r#"root ::= "x""#, "root")?, + LlamaSampler::greedy()?, + ])?; + + assert_eq!(sampler.sample(&context, batch.n_tokens() - 1)?, think_token); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_grammar_without_its_root_reports_the_missing_root(fixture: &LlamaFixture<'_>) { + assert_eq!( + LlamaSampler::grammar(fixture.model, r#"answer ::= "x""#, "root").err(), + Some(GrammarError::RootNotFound) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_grammar_with_a_syntax_error_reports_it(fixture: &LlamaFixture<'_>) { + assert_eq!( + LlamaSampler::grammar(fixture.model, "root ::= (", "root").err(), + Some(GrammarError::GrammarRejected( + GbnfValidationError::SyntaxError + )) + ); +} diff --git a/llama-cpp-bindings-tests/tests/hidden_states.rs b/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs similarity index 94% rename from llama-cpp-bindings-tests/tests/hidden_states.rs rename to llama-cpp-bindings-tests/tests/hidden_state_extraction.rs index 9461ded5f..bbc89a34d 100644 --- a/llama-cpp-bindings-tests/tests/hidden_states.rs +++ b/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs @@ -87,6 +87,49 @@ fn hidden_state_after( .to_vec()) } +fn decode_tokens( + context: &mut LlamaContext<'_>, + tokens: &[LlamaToken], + first_position: i32, + output_on_last_token: bool, +) -> Result { + let mut batch = LlamaBatch::new(ContextLimits::of(context)); + + add_tokens(&mut batch, tokens, first_position, 0, output_on_last_token)?; + context.decode(&batch)?; + + Ok(batch.n_tokens() - 1) +} + +fn final_hidden_state_context<'fixture>( + fixture: &'fixture LlamaFixture<'_>, + embeddings: bool, +) -> Result> { + Ok(LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params) + .into_llama_context_params() + .with_embeddings(embeddings) + .with_pooling_type(LlamaPoolingType::None), + )?) +} + +fn row_hidden_state_after_enabling_embeddings( + fixture: &LlamaFixture<'_>, + state: &[LlamaToken], + row: &[LlamaToken], +) -> Result> { + let mut context = final_hidden_state_context(fixture, false)?; + + decode_tokens(&mut context, state, 0, false)?; + context.set_embeddings(true); + + let row_last_index = decode_tokens(&mut context, row, i32::try_from(state.len())?, true)?; + + Ok(context.embeddings_ith(row_last_index)?.to_vec()) +} + #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -260,75 +303,6 @@ fn a_released_fork_leaves_its_shared_state_as_an_independent_decode_would( Ok(()) } -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, -)] -fn tokenizing_without_special_token_parsing_keeps_control_markup_as_text( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let markup = "<|fim_prefix|>"; - let parsed = fixture - .model - .str_to_token(markup, AddBos::Never, ParseSpecialTokens::Always)?; - let plain = fixture - .model - .str_to_token(markup, AddBos::Never, ParseSpecialTokens::Never)?; - - assert_eq!(parsed.len(), 1); - assert!(plain.len() > 1); - assert!(!plain.contains(&parsed[0])); - - Ok(()) -} - -fn decode_tokens( - context: &mut LlamaContext<'_>, - tokens: &[LlamaToken], - first_position: i32, - output_on_last_token: bool, -) -> Result { - let mut batch = LlamaBatch::new(ContextLimits::of(context)); - - add_tokens(&mut batch, tokens, first_position, 0, output_on_last_token)?; - context.decode(&batch)?; - - Ok(batch.n_tokens() - 1) -} - -fn final_hidden_state_context<'fixture>( - fixture: &'fixture LlamaFixture<'_>, - embeddings: bool, -) -> Result> { - Ok(LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_embeddings(embeddings) - .with_pooling_type(LlamaPoolingType::None), - )?) -} - -fn row_hidden_state_after_enabling_embeddings( - fixture: &LlamaFixture<'_>, - state: &[LlamaToken], - row: &[LlamaToken], -) -> Result> { - let mut context = final_hidden_state_context(fixture, false)?; - - decode_tokens(&mut context, state, 0, false)?; - context.set_embeddings(true); - - let row_last_index = decode_tokens(&mut context, row, i32::try_from(state.len())?, true)?; - - Ok(context.embeddings_ith(row_last_index)?.to_vec()) -} - #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), n_gpu_layers = 999, diff --git a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs new file mode 100644 index 000000000..ddab5491f --- /dev/null +++ b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs @@ -0,0 +1,213 @@ +use std::io::Write; + +use anyhow::Result; +use anyhow::anyhow; +use llama_cpp_bindings::SampledToken; +use llama_cpp_bindings::SampledTokenClassifier; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::json_schema_to_grammar; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn json_schema_grammar_sampler_constrains_output_to_json(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let mut context = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let prompt = "<|im_start|>user\nWhat is 2+2? Respond with a JSON object.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; + let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + + batch.add_sequence(&tokens, 0, false)?; + + context.decode(&batch)?; + + let grammar_str = json_schema_to_grammar( + r#"{"type": "object", "properties": {"answer": {"type": "string"}}, "required": ["answer"]}"#, + )?; + + let mut sampler = LlamaSampler::chain_simple([ + LlamaSampler::grammar(model, &grammar_str, "root")?, + LlamaSampler::temp(0.8)?, + LlamaSampler::greedy()?, + ])?; + + let mut classifier = SampledTokenClassifier::undeterminable(model); + let mut outcomes = Vec::new(); + let turn = classifier.sample(&mut sampler, &context, batch.n_tokens() - 1, &mut outcomes)?; + classifier.finish(&mut outcomes)?; + + assert_eq!( + outcomes.len(), + 1, + "expected one finalised outcome after finishing" + ); + let outcome = &outcomes[0]; + + let raw_as_sampled = SampledToken::Content(turn.token); + assert!( + !model.is_eog_token(&raw_as_sampled), + "Grammar sampler should not allow EOS as first token" + ); + + let piece = outcome.piece.raw(); + + assert!( + piece.starts_with('{'), + "JSON schema grammar should constrain first token to start with '{{', got: '{piece}'" + ); + assert_eq!( + classifier.usage().completion_tokens(), + 1, + "exactly one completion token sampled" + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn json_schema_constrains_output(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let backend = fixture.backend; + + let prompt = "The weather in Paris is sunny and 22 degrees. Extract as JSON:\n"; + + let mut ctx = LlamaContext::from_model( + model, + backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let tokens_list = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; + + let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); + let last_index = i32::try_from(tokens_list.len())? - 1; + + for (index, token) in (0_i32..).zip(&tokens_list) { + batch.add( + &SampledToken::Content(*token), + index, + &[0], + index == last_index, + )?; + } + + ctx.decode(&batch)?; + + let schema = r#"{ + "type": "object", + "properties": { + "city": { "type": "string" }, + "temperature": { "type": "number" } + }, + "required": ["city", "temperature"] +}"#; + + let llg_sampler = LlamaSampler::llguidance(model, "json", schema)?; + let mut sampler = LlamaSampler::chain_simple([llg_sampler, LlamaSampler::greedy()?])?; + + let mut n_cur = batch.n_tokens(); + let mut decoder = encoding_rs::UTF_8.new_decoder(); + let mut generated = String::new(); + + while n_cur <= 128 { + let token = SampledToken::Content(sampler.sample(&ctx, batch.n_tokens() - 1)?); + + if model.is_eog_token(&token) { + break; + } + + let output_string = model.token_to_piece(&token, &mut decoder, true, None)?; + generated.push_str(&output_string); + print!("{output_string}"); + std::io::stdout().flush()?; + + batch.clear(); + batch.add(&token, n_cur, &[0], true)?; + n_cur += 1; + ctx.decode(&batch)?; + } + + println!(); + + let parsed = serde_json::Deserializer::from_str(&generated) + .into_iter::() + .next() + .ok_or_else(|| anyhow!("model produced no JSON value"))??; + + assert!(parsed.get("city").is_some()); + assert!(parsed.get("temperature").is_some()); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/kv_cache_memory_availability.rs b/llama-cpp-bindings-tests/tests/kv_cache_memory_availability.rs new file mode 100644 index 000000000..a19d961c4 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/kv_cache_memory_availability.rs @@ -0,0 +1,85 @@ +use std::num::NonZeroU8; + +use anyhow::Result; +use anyhow::anyhow; +use llama_cpp_bindings::ClearKvCacheSeqError; +use llama_cpp_bindings::CopyKvCacheSeqError; +use llama_cpp_bindings::KvCacheSeqAddError; +use llama_cpp_bindings::KvCacheSeqDivError; +use llama_cpp_bindings::KvCacheSeqPosMaxError; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, + embeddings = true, +)] +fn an_encoder_context_keeps_no_memory(fixture: &LlamaFixture<'_>) -> Result<()> { + assert!(!fixture.build_context()?.has_memory()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, + embeddings = true, +)] +fn a_causal_embedding_context_keeps_memory(fixture: &LlamaFixture<'_>) -> Result<()> { + assert!(fixture.build_context()?.has_memory()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + n_threads_batch = 8, + embeddings = true, +)] +fn kv_cache_operations_respect_an_embedding_context_without_memory( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + context.clear_kv_cache(); + assert_eq!(context.clear_kv_cache_seq(Some(0), None, None), Ok(())); + context.kv_cache_seq_keep(0); + + assert_eq!( + context.copy_kv_cache_seq(0, 1, None, None), + Err(CopyKvCacheSeqError::MemoryHandleUnavailable) + ); + assert_eq!( + context.kv_cache_seq_add(0, None, None, 1), + Err(KvCacheSeqAddError::MemoryHandleUnavailable) + ); + let divisor = NonZeroU8::new(2).ok_or_else(|| anyhow!("2 is non-zero"))?; + assert_eq!( + context.kv_cache_seq_div(0, None, None, divisor), + Err(KvCacheSeqDivError::MemoryHandleUnavailable) + ); + assert_eq!( + context.kv_cache_seq_pos_max(0), + Err(KvCacheSeqPosMaxError::MemoryHandleUnavailable) + ); + assert!(matches!( + context.clear_kv_cache_seq(Some(u32::MAX), None, None), + Err(ClearKvCacheSeqError::SeqIdTooLarge(_)) + )); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/kv_cache_occupancy.rs b/llama-cpp-bindings-tests/tests/kv_cache_occupancy.rs new file mode 100644 index 000000000..1d68ce287 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/kv_cache_occupancy.rs @@ -0,0 +1,133 @@ +use anyhow::Result; +use llama_cpp_bindings_tests::prime_kv_cache::prime_kv_cache; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn clear_kv_cache_resets_positions(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_kv_cache(fixture, &mut context)?; + + context.clear_kv_cache(); + assert_eq!(context.kv_cache_seq_pos_max(0)?, -1); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn kv_cache_seq_pos_max_is_non_negative_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_kv_cache(fixture, &mut context)?; + + assert!(context.kv_cache_seq_pos_max(0)? >= 0); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn kv_cache_seq_keep_retains_specified_sequence(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_kv_cache(fixture, &mut context)?; + + context.kv_cache_seq_keep(0); + + assert!(context.kv_cache_seq_pos_max(0)? >= 0); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/kv_cache_position_bounds.rs b/llama-cpp-bindings-tests/tests/kv_cache_position_bounds.rs new file mode 100644 index 000000000..afa35c52b --- /dev/null +++ b/llama-cpp-bindings-tests/tests/kv_cache_position_bounds.rs @@ -0,0 +1,388 @@ +use std::num::NonZeroU8; + +use anyhow::Result; +use anyhow::anyhow; +use llama_cpp_bindings::error::ClearKvCacheSeqError; +use llama_cpp_bindings::error::CopyKvCacheSeqError; +use llama_cpp_bindings::error::KvCacheSeqAddError; +use llama_cpp_bindings::error::KvCacheSeqDivError; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn copy_kv_cache_seq_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let result = context.copy_kv_cache_seq(0, 1, Some(u32::MAX), None); + + assert_eq!( + result.unwrap_err(), + CopyKvCacheSeqError::P0TooLarge( + i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") + ) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn copy_kv_cache_seq_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let result = context.copy_kv_cache_seq(0, 1, Some(0), Some(u32::MAX)); + + assert_eq!( + result.unwrap_err(), + CopyKvCacheSeqError::P1TooLarge( + i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") + ) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn clear_kv_cache_seq_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let result = context.clear_kv_cache_seq(Some(0), Some(u32::MAX), None); + + assert_eq!( + result.unwrap_err(), + ClearKvCacheSeqError::P0TooLarge( + i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") + ) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn clear_kv_cache_seq_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let result = context.clear_kv_cache_seq(Some(0), Some(0), Some(u32::MAX)); + + assert_eq!( + result.unwrap_err(), + ClearKvCacheSeqError::P1TooLarge( + i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") + ) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn kv_cache_seq_add_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let result = context.kv_cache_seq_add(0, Some(u32::MAX), None, 1); + + assert_eq!( + result.unwrap_err(), + KvCacheSeqAddError::P0TooLarge( + i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") + ) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn kv_cache_seq_add_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let result = context.kv_cache_seq_add(0, Some(0), Some(u32::MAX), 1); + + assert_eq!( + result.unwrap_err(), + KvCacheSeqAddError::P1TooLarge( + i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") + ) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn kv_cache_seq_div_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let divisor = NonZeroU8::new(2).ok_or_else(|| anyhow!("2 is non-zero"))?; + let result = context.kv_cache_seq_div(0, Some(u32::MAX), None, divisor); + + assert_eq!( + result.unwrap_err(), + KvCacheSeqDivError::P0TooLarge( + i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") + ) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn kv_cache_seq_div_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let divisor = NonZeroU8::new(2).ok_or_else(|| anyhow!("2 is non-zero"))?; + let result = context.kv_cache_seq_div(0, Some(0), Some(u32::MAX), divisor); + + assert_eq!( + result.unwrap_err(), + KvCacheSeqDivError::P1TooLarge( + i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") + ) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/kv_cache_position_shift_rope_compatibility.rs b/llama-cpp-bindings-tests/tests/kv_cache_position_shift_rope_compatibility.rs new file mode 100644 index 000000000..9d9c81e7a --- /dev/null +++ b/llama-cpp-bindings-tests/tests/kv_cache_position_shift_rope_compatibility.rs @@ -0,0 +1,113 @@ +use std::num::NonZeroU8; + +use anyhow::Result; +use anyhow::anyhow; +use llama_cpp_bindings::error::KvCacheSeqAddError; +use llama_cpp_bindings::error::KvCacheSeqDivError; +use llama_cpp_bindings_tests::prime_kv_cache::prime_kv_cache; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn kv_cache_seq_add_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_kv_cache(fixture, &mut context)?; + + let result = context.kv_cache_seq_add(0, Some(0), None, 1); + + assert_eq!( + result.unwrap_err(), + KvCacheSeqAddError::IncompatibleRopeType + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn kv_cache_seq_div_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_kv_cache(fixture, &mut context)?; + + let divisor = NonZeroU8::new(2).ok_or_else(|| anyhow!("2 is non-zero"))?; + let result = context.kv_cache_seq_div(0, Some(0), None, divisor); + + assert_eq!( + result.unwrap_err(), + KvCacheSeqDivError::IncompatibleRopeType + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn kv_cache_seq_add_succeeds_on_embedding_model(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_kv_cache(fixture, &mut context)?; + + let result = context.kv_cache_seq_add(0, Some(0), None, 1); + + assert!(result.is_ok()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn kv_cache_seq_div_succeeds_on_embedding_model(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_kv_cache(fixture, &mut context)?; + + let divisor = NonZeroU8::new(2).ok_or_else(|| anyhow!("2 is non-zero"))?; + let result = context.kv_cache_seq_div(0, Some(0), None, divisor); + + assert!(result.is_ok()); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/kv_cache_priming.rs b/llama-cpp-bindings-tests/tests/kv_cache_priming.rs new file mode 100644 index 000000000..a973a21e4 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/kv_cache_priming.rs @@ -0,0 +1,45 @@ +use anyhow::Result; +use llama_cpp_bindings::StringToTokenError; +use llama_cpp_bindings::batch_add_error::BatchAddError; +use llama_cpp_bindings_tests::prime_kv_cache_with::prime_kv_cache_with; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 256, + n_ubatch = 64, +)] +fn prime_kv_cache_surfaces_each_underlying_error(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + assert!( + prime_kv_cache_with(fixture, &mut context, "Hello\0world") + .is_err_and(|error| error.downcast_ref::().is_some()), + "an interior null byte must surface a tokenization error" + ); + assert_eq!( + prime_kv_cache_with(fixture, &mut context, &"word ".repeat(300)) + .err() + .and_then(|error| error.downcast_ref::().cloned()), + Some(BatchAddError::BatchFull { n_batch: 256 }), + "more tokens than one decode takes must surface an add-sequence error" + ); + + let filler = "word ".repeat(40); + let mut decode_result = prime_kv_cache_with(fixture, &mut context, &filler); + let mut attempts = 0; + while decode_result.is_ok() && attempts < 16 { + decode_result = prime_kv_cache_with(fixture, &mut context, &filler); + attempts += 1; + } + assert!( + decode_result.is_err(), + "filling the context past its window must surface a decode error" + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/kv_cache_sequence_edits.rs b/llama-cpp-bindings-tests/tests/kv_cache_sequence_edits.rs new file mode 100644 index 000000000..2c9a5c156 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/kv_cache_sequence_edits.rs @@ -0,0 +1,133 @@ +use anyhow::Result; +use llama_cpp_bindings_tests::prime_kv_cache::prime_kv_cache; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn clear_kv_cache_seq_accepts_an_explicit_range(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_kv_cache(fixture, &mut context)?; + + assert_eq!( + context.clear_kv_cache_seq(Some(0), Some(0), Some(1)), + Ok(()) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn copy_kv_cache_seq_succeeds(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_kv_cache(fixture, &mut context)?; + + assert_eq!(context.copy_kv_cache_seq(0, 1, None, None), Ok(())); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn copy_kv_cache_seq_accepts_an_explicit_range(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_kv_cache(fixture, &mut context)?; + + assert_eq!(context.copy_kv_cache_seq(0, 2, Some(0), Some(1)), Ok(())); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/kv_cache_sequence_id_bounds.rs b/llama-cpp-bindings-tests/tests/kv_cache_sequence_id_bounds.rs new file mode 100644 index 000000000..879cba9d8 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/kv_cache_sequence_id_bounds.rs @@ -0,0 +1,98 @@ +use anyhow::Result; +use llama_cpp_bindings::error::ClearKvCacheSeqError; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn kv_cache_seq_pos_max_rejects_out_of_range_sequence(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = fixture.build_context()?; + + let result = context.kv_cache_seq_pos_max(999); + + assert_eq!( + result, + Err(llama_cpp_bindings::KvCacheSeqPosMaxError::SequenceIdOutOfRange { seq_id: 999 }) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn clear_kv_cache_seq_rejects_sequence_id_exceeding_i32_max( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + let result = context.clear_kv_cache_seq(Some(u32::MAX), None, None); + + assert_eq!( + result.unwrap_err(), + ClearKvCacheSeqError::SeqIdTooLarge( + i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") + ) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs b/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs new file mode 100644 index 000000000..a102fd17a --- /dev/null +++ b/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs @@ -0,0 +1,138 @@ +use std::ffi::CString; + +use anyhow::Result; +use llama_cpp_bindings::GrammarError; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn grammar_lazy_returns_sampler_for_valid_grammar_with_patterns(fixture: &LlamaFixture<'_>) { + let patterns = vec!["\\{.*".to_owned()]; + let sampler = + LlamaSampler::grammar_lazy(fixture.model, "root ::= \"hello\"", "root", &patterns, &[]); + + assert!(sampler.is_ok()); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn grammar_lazy_with_root_not_found_returns_error(fixture: &LlamaFixture<'_>) { + let patterns = vec!["function".to_owned()]; + let result = + LlamaSampler::grammar_lazy(fixture.model, "expr ::= \"hello\"", "root", &patterns, &[]); + + assert_eq!(result.err(), Some(GrammarError::RootNotFound)); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn grammar_lazy_with_null_byte_in_pattern_returns_error(fixture: &LlamaFixture<'_>) { + let patterns = vec!["hel\0lo".to_owned()]; + let result = + LlamaSampler::grammar_lazy(fixture.model, "root ::= \"hello\"", "root", &patterns, &[]); + + assert_eq!( + result.err(), + Some(GrammarError::TriggerPatternContainsNul( + CString::new("hel\0lo").expect_err("the pattern carries a nul byte") + )) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn grammar_lazy_returns_sampler_for_valid_grammar_with_trigger_tokens( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let trigger_tokens = + fixture + .model + .str_to_token("{", AddBos::Never, ParseSpecialTokens::Always)?; + + assert!( + !trigger_tokens.is_empty(), + "the tokenizer must produce at least one trigger token" + ); + + let sampler = LlamaSampler::grammar_lazy( + fixture.model, + "root ::= \"hello\"", + "root", + &[], + &trigger_tokens, + ); + + assert!( + sampler.is_ok(), + "trigger tokens alone must be enough to build a lazy grammar sampler" + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn grammar_lazy_with_null_byte_in_grammar_returns_error(fixture: &LlamaFixture<'_>) { + let result = + LlamaSampler::grammar_lazy(fixture.model, "root ::= \"hel\0lo\"", "root", &[], &[]); + + assert_eq!( + result.err(), + Some(GrammarError::GrammarContainsNul( + CString::new("root ::= \"hel\0lo\"").expect_err("the grammar carries a nul byte") + )) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn grammar_lazy_with_malformed_regex_returns_invalid_trigger_pattern(fixture: &LlamaFixture<'_>) { + let patterns = vec!["[".to_owned()]; + let result = + LlamaSampler::grammar_lazy(fixture.model, "root ::= \"hello\"", "root", &patterns, &[]); + + assert!(matches!( + result, + Err(GrammarError::InvalidTriggerPattern { .. }), + )); +} diff --git a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs new file mode 100644 index 000000000..f57806cb5 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs @@ -0,0 +1,599 @@ +use std::ffi::CStr; + +use anyhow::Context as _; +use anyhow::Result; +use anyhow::anyhow; +use llama_cpp_bindings::SamplerAcceptError; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::llguidance_sampler::create_llg_sampler; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +const JSON_SCHEMA: &str = + r#"{"type":"object","properties":{"answer":{"type":"string"}},"required":["answer"]}"#; +const REGEX_GRAMMAR: &str = r"yes|no"; +const LARK_GRAMMAR: &str = r#"start: "yes" | "no""#; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn llguidance_method_creates_sampler(fixture: &LlamaFixture<'_>) { + let result = LlamaSampler::llguidance(fixture.model, "regex", r"yes|no"); + + assert!(result.is_ok()); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn creates_sampler_with_valid_json_schema(fixture: &LlamaFixture<'_>) -> Result<()> { + let sampler = create_llg_sampler(fixture.model, "json", JSON_SCHEMA)?; + + assert!(!sampler.as_ptr().is_null()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn creates_sampler_with_valid_regex_grammar(fixture: &LlamaFixture<'_>) -> Result<()> { + let sampler = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; + + assert!(!sampler.as_ptr().is_null()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn creates_sampler_with_valid_lark_grammar(fixture: &LlamaFixture<'_>) -> Result<()> { + let sampler = create_llg_sampler(fixture.model, "lark", LARK_GRAMMAR)?; + + assert!(!sampler.as_ptr().is_null()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn returns_error_for_unknown_grammar_kind(fixture: &LlamaFixture<'_>) { + let result = create_llg_sampler(fixture.model, "not_a_real_kind", "anything"); + + assert!(result.is_err()); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn returns_error_for_malformed_json_schema(fixture: &LlamaFixture<'_>) { + let result = create_llg_sampler(fixture.model, "json", "{this is not valid json"); + + assert!(result.is_err()); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn returns_error_for_malformed_regex(fixture: &LlamaFixture<'_>) { + let result = create_llg_sampler(fixture.model, "regex", "[invalid"); + + assert!(result.is_err()); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn name_callback_returns_llguidance(fixture: &LlamaFixture<'_>) -> Result<()> { + let sampler = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; + + let name_ptr = unsafe { llama_cpp_bindings_sys::llama_sampler_name(sampler.as_ptr()) }; + assert!(!name_ptr.is_null()); + let name = unsafe { CStr::from_ptr(name_ptr) }.to_str()?; + + assert_eq!(name, "llguidance"); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn clone_via_ffi_creates_independent_sampler(fixture: &LlamaFixture<'_>) -> Result<()> { + let sampler = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; + + let cloned = unsafe { llama_cpp_bindings_sys::llama_sampler_clone(sampler.as_ptr()) }; + + assert!(!cloned.is_null()); + + unsafe { llama_cpp_bindings_sys::llama_sampler_free(cloned) }; + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn samples_token_constrained_by_grammar(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let backend = fixture.backend; + let mut context = LlamaContext::from_model( + model, + backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let prompt = "Answer yes or no:"; + let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let llg_sampler = create_llg_sampler(model, "regex", REGEX_GRAMMAR)?; + let mut chain = LlamaSampler::chain_simple([llg_sampler, LlamaSampler::greedy()?])?; + + let token = chain.sample(&context, batch.n_tokens() - 1)?; + assert!( + token.0 >= 0, + "grammar-constrained sampling must yield a valid token id without the grammar rejecting it" + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn reset_rolls_back_accepted_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut sampler = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; + + let yes_tokens = + fixture + .model + .str_to_token("yes", AddBos::Never, ParseSpecialTokens::Always)?; + let first_allowed_token = *yes_tokens + .first() + .ok_or_else(|| anyhow!("the tokenizer must produce a token for \"yes\""))?; + + sampler + .accept(first_allowed_token) + .context("the grammar must accept the first token of an allowed word")?; + + sampler + .reset() + .context("reset must roll a healthy grammar matcher back to its initial state")?; + + assert_eq!( + sampler.accept(first_allowed_token), + Ok(()), + "after reset the same token must be acceptable again, which is what rollback means" + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn accept_maps_an_out_of_vocabulary_token_to_grammar_callback_failure( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut sampler = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; + + let huge_token = LlamaToken(i32::MAX - 1); + let result = sampler.accept(huge_token); + + let Err(SamplerAcceptError::GrammarCallbackFailed { message }) = result else { + panic!("expected a grammar callback failure, got {result:?}"); + }; + assert!(message.contains("token id 2147483646 out of range")); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn approximate_tok_env_drives_consistent_grammar_constraint( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let first = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; + let second = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; + + assert!(!first.as_ptr().is_null()); + assert!(!second.as_ptr().is_null()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn llguidance_chain_samples_a_valid_token(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let backend = fixture.backend; + let mut context = LlamaContext::from_model( + model, + backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let tokens = model.str_to_token("Answer:", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let llg_sampler = create_llg_sampler(model, "regex", REGEX_GRAMMAR)?; + let mut chain = LlamaSampler::chain_simple([llg_sampler, LlamaSampler::greedy()?])?; + let token = chain.sample(&context, batch.n_tokens() - 1)?; + + assert!(token.0 >= 0); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs b/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs new file mode 100644 index 000000000..68305a8b1 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs @@ -0,0 +1,17 @@ +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn logit_bias_with_empty_biases_succeeds(_fixture: &LlamaFixture<'_>) { + let result = LlamaSampler::logit_bias(0, &[]); + + assert!(result.is_ok()); +} diff --git a/llama-cpp-bindings-tests/tests/logits_retrieval.rs b/llama-cpp-bindings-tests/tests/logits_retrieval.rs new file mode 100644 index 000000000..615a10a09 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/logits_retrieval.rs @@ -0,0 +1,91 @@ +use anyhow::Result; +use llama_cpp_bindings::LogitsError; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn get_logits_returns_values_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + + let decode_result = context.decode(&batch); + assert!(decode_result.is_ok()); + + let logits = context.get_logits()?; + assert_ne!(logits, Vec::::new()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn get_logits_ith_returns_valid_slice(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let last_index = i32::try_from(tokens.len() - 1)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let logits = context.get_logits_ith(last_index)?; + + assert_eq!(logits.len(), usize::try_from(fixture.model.n_vocab())?); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn get_logits_ith_returns_token_not_initialized_for_unknown_index( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let result = context.get_logits_ith(7); + + assert_eq!(result, Err(LogitsError::TokenNotInitialized(7))); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/lora_adapters.rs b/llama-cpp-bindings-tests/tests/lora_adapters.rs new file mode 100644 index 000000000..6469728eb --- /dev/null +++ b/llama-cpp-bindings-tests/tests/lora_adapters.rs @@ -0,0 +1,63 @@ +use anyhow::Result; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::model::lora_adapter_scale::LoraAdapterScale; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::download_model::download_model; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace( + "Qwen/Qwen2.5-7B-Instruct-GGUF", + "qwen2.5-7b-instruct-q2_k.gguf" + ), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn set_lora_adapters_applies_a_real_adapter(fixture: &LlamaFixture<'_>) -> Result<()> { + let adapter_path = download_model( + "ggml-org/LoRA-Deepthink-Reasoning-Qwen2.5-7B-Instruct-Q8_0-GGUF", + "Deepthink-Reasoning-Adapter-q8_0.gguf", + )?; + let adapter = fixture.model.lora_adapter_init(&adapter_path)?; + let context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + assert_eq!( + context.set_lora_adapters(&[LoraAdapterScale { + adapter: &adapter, + scale: 1.0, + }]), + Ok(()) + ); + + assert_eq!(context.set_lora_adapters(&[]), Ok(())); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn set_lora_adapters_accepts_an_empty_adapter_set(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let result = context.set_lora_adapters(&[]); + + assert!(result.is_ok()); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/main.rs b/llama-cpp-bindings-tests/tests/main.rs index 52cb51445..c0f25892e 100644 --- a/llama-cpp-bindings-tests/tests/main.rs +++ b/llama-cpp-bindings-tests/tests/main.rs @@ -1,25 +1,86 @@ -#![expect( - clippy::unnecessary_wraps, - reason = "harness trial functions must return Result to satisfy LlamaTestFn registration" -)] +mod approximate_tok_env_eos_selection; mod backend_initialization; mod batch_context_limits; -mod chat_protocol; +mod chat_answer_parsing; +mod chat_prompt_rendering; +mod chat_session_input_validation; +mod chat_template_lookup; +mod chat_template_tool_call_support; +mod completion_usage_accounting; mod context_after_decoding_text; -mod context_state; +mod context_creation_limits; +mod context_dimensions; +mod context_synchronization; +mod debug_formatting; +mod decode_abort_control; +mod decode_batch_rejection; +mod detokenization; +mod dry_sampler; +mod duck_typed_tool_calls; mod embedding_availability; -mod embedding_models; +mod embedding_generation; +mod embedding_retrieval_errors; mod embedding_row_width; -mod generation_control; -mod hidden_states; +mod encode_batch_rejection; +mod encoder_requirement; +mod end_of_generation_token_detection; +mod generation_end_classification; +mod grammar_sampler; +mod hidden_state_extraction; +mod json_schema_constrained_generation; +mod kv_cache_memory_availability; +mod kv_cache_occupancy; +mod kv_cache_position_bounds; +mod kv_cache_position_shift_rope_compatibility; +mod kv_cache_priming; +mod kv_cache_sequence_edits; +mod kv_cache_sequence_id_bounds; +mod lazy_grammar_sampler; +mod llguidance_sampler; +mod logit_bias_sampler; mod logits_availability; +mod logits_retrieval; +mod lora_adapters; +mod model_architecture; mod model_derived_data_caching; -mod model_introspection; +mod model_hyperparameters; mod model_loading_errors; +mod model_metadata; mod multimodal_audio; +mod multimodal_bitmap_construction; +mod multimodal_chunk_copy; +mod multimodal_chunk_encoding; +mod multimodal_chunk_evaluation; +mod multimodal_chunk_tokenization; mod multimodal_fusion; +mod multimodal_projector_capabilities; +mod multimodal_projector_loading; mod multimodal_prompt_parts_tokenization; -mod multimodal_vision; -mod structured_chat_output; +mod multimodal_reasoning_classification; +mod multimodal_usage_accounting; +mod multimodal_vision_inference; +mod parameter_fitting; +mod performance_timings; +mod prompt_token_accounting; +mod reasoning_classification; +mod reranking; +mod sampled_token_ingestion; +mod sampler_token_acceptance; +mod session_file_path_validation; +mod session_file_token_capacity; +mod session_files; +mod special_tokens; +mod state_snapshots; +mod threadpool_detachment; +mod token_attributes; +mod token_candidates; +mod tokenization; +mod tokenize_synthetic; +mod tool_call_classification; +mod tool_call_parsing; +mod unconstrained_token_sampling; +mod vocabulary_bounds; +mod vocabulary_enumeration; +mod vocabulary_type; llama_cpp_test_harness::llama_tests_main!(); diff --git a/llama-cpp-bindings-tests/tests/model_architecture.rs b/llama-cpp-bindings-tests/tests/model_architecture.rs new file mode 100644 index 000000000..9c614efee --- /dev/null +++ b/llama-cpp-bindings-tests/tests/model_architecture.rs @@ -0,0 +1,136 @@ +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn is_recurrent_returns_false_for_transformer(fixture: &LlamaFixture<'_>) { + assert!(!fixture.model.is_recurrent()); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn is_hybrid_returns_false_for_non_hybrid_default_models(fixture: &LlamaFixture<'_>) { + assert!( + !fixture.model.is_hybrid(), + "DeepSeek-R1-Distill-Llama-8B and GLM-4.7-Flash are pure transformers, not hybrid; got is_hybrid=true" + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn is_hybrid_returns_true_for_qwen_default_models(fixture: &LlamaFixture<'_>) { + assert!( + fixture.model.is_hybrid(), + "Qwen 3.5 and Qwen 3.6 default GGUFs are reported as hybrid by llama.cpp; got is_hybrid=false" + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn rope_type_returns_a_known_variant_for_rope_carrying_default_models(fixture: &LlamaFixture<'_>) { + use llama_cpp_bindings::model::rope_type::RopeType; + let rope = fixture.model.rope_type(); + assert!( + matches!( + rope, + Some(RopeType::Norm | RopeType::NeoX | RopeType::MRope | RopeType::Vision) + ), + "rope_type must be a known variant for DeepSeek and GLM-4.7; got {rope:?}" + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn rope_type_returns_none_for_qwen_default_models(fixture: &LlamaFixture<'_>) { + let rope = fixture.model.rope_type(); + assert!( + rope.is_none(), + "Qwen 3.5 and Qwen 3.6 default GGUFs do not expose a rope_type in their metadata; got {rope:?}" + ); +} diff --git a/llama-cpp-bindings-tests/tests/model_derived_data_caching.rs b/llama-cpp-bindings-tests/tests/model_derived_data_caching.rs index 59bacd40a..0dc760f79 100644 --- a/llama-cpp-bindings-tests/tests/model_derived_data_caching.rs +++ b/llama-cpp-bindings-tests/tests/model_derived_data_caching.rs @@ -1,7 +1,6 @@ use std::ptr; use std::sync::Arc; -use anyhow::Context; use anyhow::Result; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -31,32 +30,9 @@ fn approximate_tok_env_is_cached_across_calls(fixture: &LlamaFixture<'_>) -> Res n_batch = 512, n_ubatch = 128 )] -fn streaming_markers_are_cached_across_calls(fixture: &LlamaFixture<'_>) -> Result<()> { - let first = fixture.model.streaming_markers()?; - let second = fixture.model.streaming_markers()?; - - assert!(Arc::ptr_eq(&first, &second)); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128 -)] -fn reasoning_markers_are_cached_across_calls(fixture: &LlamaFixture<'_>) -> Result<()> { - let first = fixture - .model - .reasoning_markers()? - .context("Qwen3.5 must expose reasoning markers")?; - let second = fixture - .model - .reasoning_markers()? - .context("Qwen3.5 must expose reasoning markers")?; +fn chat_templates_are_prepared_once_per_model(fixture: &LlamaFixture<'_>) -> Result<()> { + let first = fixture.model.chat_templates()?; + let second = fixture.model.chat_templates()?; assert!(ptr::eq(first, second)); diff --git a/llama-cpp-bindings-tests/tests/model_hyperparameters.rs b/llama-cpp-bindings-tests/tests/model_hyperparameters.rs new file mode 100644 index 000000000..fcf7f6e22 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/model_hyperparameters.rs @@ -0,0 +1,193 @@ +use anyhow::Result; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn model_reports_positive_core_hyperparameters(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + + assert!(model.n_vocab() > 0); + assert!(model.n_embd() > 0); + assert!(model.n_params() > 0); + assert!(model.n_ctx_train()? > 0); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn n_layer_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { + assert!(fixture.model.n_layer()? > 0); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn n_head_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { + assert!(fixture.model.n_head()? > 0); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn n_head_kv_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { + assert!(fixture.model.n_head_kv()? > 0); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn model_size_returns_nonzero(fixture: &LlamaFixture<'_>) { + assert!(fixture.model.size() > 0); +} diff --git a/llama-cpp-bindings-tests/tests/model_introspection.rs b/llama-cpp-bindings-tests/tests/model_introspection.rs deleted file mode 100644 index 40f87f29d..000000000 --- a/llama-cpp-bindings-tests/tests/model_introspection.rs +++ /dev/null @@ -1,1855 +0,0 @@ -use std::ffi::CString; -use std::num::NonZeroU16; -use std::num::NonZeroU32; -use std::pin::pin; - -use anyhow::Context; -use anyhow::Result; -use anyhow::anyhow; -use llama_cpp_bindings::SampledToken; -use llama_cpp_bindings::TokenAttrError; -use llama_cpp_bindings::TokenOutsideVocabulary; -use llama_cpp_bindings::TokenToStringError; -use llama_cpp_bindings::context::params::LlamaContextParams; -use llama_cpp_bindings::max_devices; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; -use llama_cpp_bindings::model::params::LlamaModelParams; -use llama_cpp_bindings::model::params::fit_extra_model::FitExtraModel; -use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn model_loads_with_valid_metadata(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - - assert!(model.n_vocab() > 0); - assert!(model.n_embd() > 0); - assert!(model.n_params() > 0); - assert!(model.n_ctx_train()? > 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn n_layer_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(fixture.model.n_layer()? > 0); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn n_head_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(fixture.model.n_head()? > 0); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn n_head_kv_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(fixture.model.n_head_kv()? > 0); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn model_size_returns_nonzero(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(fixture.model.size() > 0); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn is_recurrent_returns_false_for_transformer(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(!fixture.model.is_recurrent()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn is_hybrid_returns_false_for_non_hybrid_default_models(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!( - !fixture.model.is_hybrid(), - "DeepSeek-R1-Distill-Llama-8B and GLM-4.7-Flash are pure transformers, not hybrid; got is_hybrid=true" - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn is_hybrid_returns_true_for_qwen_default_models(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!( - fixture.model.is_hybrid(), - "Qwen 3.5 and Qwen 3.6 default GGUFs are reported as hybrid by llama.cpp; got is_hybrid=false" - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn rope_type_returns_a_known_variant_for_rope_carrying_default_models( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - use llama_cpp_bindings::model::rope_type::RopeType; - let rope = fixture.model.rope_type(); - assert!( - matches!( - rope, - Some(RopeType::Norm | RopeType::NeoX | RopeType::MRope | RopeType::Vision) - ), - "rope_type must be a known variant for DeepSeek and GLM-4.7; got {rope:?}" - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn rope_type_returns_none_for_qwen_default_models(fixture: &LlamaFixture<'_>) -> Result<()> { - let rope = fixture.model.rope_type(); - assert!( - rope.is_none(), - "Qwen 3.5 and Qwen 3.6 default GGUFs do not expose a rope_type in their metadata; got {rope:?}" - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn default_models_use_a_byte_pair_encoding_vocabulary(fixture: &LlamaFixture<'_>) -> Result<()> { - use llama_cpp_bindings::model::vocab_type::VocabType; - - assert_eq!(fixture.model.vocab_type()?, VocabType::Bpe); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn meta_count_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(fixture.model.meta_count() > 0); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn meta_key_by_index_returns_valid_key(fixture: &LlamaFixture<'_>) -> Result<()> { - let key = fixture.model.meta_key_by_index(0)?; - assert_ne!(key, ""); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn meta_val_str_by_index_returns_valid_value(fixture: &LlamaFixture<'_>) -> Result<()> { - let value = fixture.model.meta_val_str_by_index(0)?; - assert_ne!(value, ""); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn meta_key_by_index_out_of_range_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = fixture.model.meta_key_by_index(999_999); - assert!(result.is_err()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn meta_val_str_by_index_out_of_range_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = fixture.model.meta_val_str_by_index(999_999); - assert!(result.is_err()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn meta_val_str_returns_value_for_known_key(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let first_key = model.meta_key_by_index(0)?; - let value = model.meta_val_str(&first_key)?; - assert_ne!(value, ""); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn meta_val_str_with_long_value_triggers_buffer_resize(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let count = model.meta_count(); - - for index in 0..count { - let key = model.meta_key_by_index(index); - let value = model.meta_val_str_by_index(index); - assert!(key.is_ok()); - assert!(value.is_ok()); - } - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn meta_val_str_with_null_byte_in_key_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = fixture.model.meta_val_str("key\0with_null"); - assert!(result.is_err()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn fit_params_succeeds_with_test_model(fixture: &LlamaFixture<'_>) -> Result<()> { - let model_path_utf8 = fixture - .model_path - .to_str() - .ok_or_else(|| anyhow!("model path is not valid UTF-8"))?; - let model_path_c = CString::new(model_path_utf8)?; - - let mut params = pin!(LlamaModelParams::default()); - let mut context_params = LlamaContextParams::default(); - let mut margins = vec![0usize; max_devices()]; - - let result = params.as_mut().fit_params( - &model_path_c, - &mut context_params, - &mut margins, - 512, - None, - llama_cpp_bindings_sys::GGML_LOG_LEVEL_NONE, - ); - - let fit = result.map_err(|fit_error| anyhow!("fit_params failed: {fit_error:?}"))?; - assert!(fit.n_ctx > 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn fit_params_aligns_the_extra_model_context_with_the_fitted_one( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model_path_utf8 = fixture - .model_path - .to_str() - .ok_or_else(|| anyhow!("model path is not valid UTF-8"))?; - let model_path_c = CString::new(model_path_utf8)?; - - let mut params = pin!(LlamaModelParams::default()); - let mut context_params = LlamaContextParams::default(); - let mut margins = vec![0usize; max_devices()]; - - let unfittable_extra_n_ctx = NonZeroU32::MIN; - let mut extra_params = pin!(LlamaModelParams::default()); - let mut extra_context_params = - LlamaContextParams::default().with_n_ctx(Some(unfittable_extra_n_ctx)); - let mut extra = FitExtraModel { - model_path: &model_path_c, - model_params: extra_params.as_mut(), - context_params: &mut extra_context_params, - shares_model: true, - }; - - let result = params.as_mut().fit_params( - &model_path_c, - &mut context_params, - &mut margins, - 512, - Some(&mut extra), - llama_cpp_bindings_sys::GGML_LOG_LEVEL_NONE, - ); - - let fit = result.map_err(|fit_error| anyhow!("fit_params failed: {fit_error:?}"))?; - assert!(fit.n_ctx > 0); - assert_ne!( - extra_context_params.n_ctx(), - Some(unfittable_extra_n_ctx), - "the llama.cpp fit must overwrite the extra model's context size" - ); - assert_eq!( - extra_context_params.context_params.n_ctx, - context_params.context_params.n_ctx - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn special_tokens_exist(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let eos = model.token_eos().context("model declares an EOS token")?; - - assert_ne!(model.token_bos(), Some(eos)); - assert!(model.is_eog_token(&SampledToken::Content(eos))); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 64, - embeddings = true, -)] -fn model_without_a_bos_token_reports_none(fixture: &LlamaFixture<'_>) -> Result<()> { - assert_eq!(fixture.model.token_bos(), None); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_nl_is_the_first_token_of_a_tokenized_newline(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let newline_tokens = model.str_to_token("\n", AddBos::Never, ParseSpecialTokens::Never)?; - - assert_eq!(model.token_nl(), newline_tokens.first().copied()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn is_eog_token_classifies_reasoning_variant(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let eos = model.token_eos().context("model declares an EOS token")?; - assert!(model.is_eog_token(&SampledToken::Reasoning(eos))); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn is_eog_token_classifies_tool_call_variant(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let eos = model.token_eos().context("model declares an EOS token")?; - assert!(model.is_eog_token(&SampledToken::ToolCall(eos))); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn is_eog_token_classifies_undeterminable_variant(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let eos = model.token_eos().context("model declares an EOS token")?; - assert!(model.is_eog_token(&SampledToken::Undeterminable(eos))); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn decoder_only_model_has_no_decoder_start_token(fixture: &LlamaFixture<'_>) -> Result<()> { - assert_eq!(fixture.model.decode_start_token(), None); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn chat_model_has_no_separator_token(fixture: &LlamaFixture<'_>) -> Result<()> { - assert_eq!(fixture.model.token_sep(), None); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_attr_returns_attrs_for_bos(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let bos = model.token_bos().context("model declares a BOS token")?; - let attrs = model.token_attr(bos)?; - let bit_repr = format!("{:?}", *attrs); - assert!( - !bit_repr.is_empty(), - "token_attr(bos) must produce Debug output" - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn str_to_token_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let tokens = model.str_to_token("hello world", AddBos::Never, ParseSpecialTokens::Always)?; - assert_ne!(tokens, Vec::::new()); - let mut decoder = encoding_rs::UTF_8.new_decoder(); - let piece = - model.token_to_piece(&SampledToken::Content(tokens[0]), &mut decoder, false, None)?; - - assert_ne!(piece, ""); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn str_to_token_grows_buffer_when_initial_estimation_too_small( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let many_short_chars = "a b c d e f g h i j k l"; - let tokens = - fixture - .model - .str_to_token(many_short_chars, AddBos::Always, ParseSpecialTokens::Always)?; - - assert!( - tokens.len() > 8, - "expected regrow; got {} tokens", - tokens.len() - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn str_to_token_with_add_bos_never(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let tokens_with_bos = - model.str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let tokens_without_bos = - model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; - - assert!(tokens_with_bos.len() >= tokens_without_bos.len()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn str_to_token_with_many_tokens_triggers_buffer_resize(fixture: &LlamaFixture<'_>) -> Result<()> { - use std::fmt::Write; - - let many_numbers = (0..2000).fold(String::new(), |mut accumulator, number| { - let _ = write!(accumulator, "{number} "); - accumulator - }); - - let tokens = - fixture - .model - .str_to_token(&many_numbers, AddBos::Always, ParseSpecialTokens::Always)?; - - assert!(tokens.len() > many_numbers.len() / 2); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_to_piece_bytes_returns_bytes_for_known_token(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let tokens = model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; - let bytes = model.token_to_piece_bytes(tokens[0], 32, false, None)?; - - assert_ne!(bytes, Vec::::new()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_to_piece_handles_large_token_requiring_buffer_resize( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let mut decoder = encoding_rs::UTF_8.new_decoder(); - - for (token, _) in model.tokens(true).take(200) { - let result = model.token_to_piece(&SampledToken::Content(token), &mut decoder, true, None); - assert!(result.is_ok()); - } - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_to_piece_bytes_insufficient_buffer_returns_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let tokens = model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; - let result = model.token_to_piece_bytes(tokens[0], 1, false, None); - - assert!( - result - .unwrap_err() - .to_string() - .contains("Insufficient Buffer Space") - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_to_piece_with_lstrip(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut decoder = encoding_rs::UTF_8.new_decoder(); - let tokens = model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; - let result = model.token_to_piece( - &SampledToken::Content(tokens[0]), - &mut decoder, - false, - NonZeroU16::new(1), - ); - - assert!(result.is_ok()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_to_piece_decodes_reasoning_variant(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut decoder = encoding_rs::UTF_8.new_decoder(); - let tokens = model.str_to_token("hi", AddBos::Never, ParseSpecialTokens::Always)?; - - let piece = model.token_to_piece( - &SampledToken::Reasoning(tokens[0]), - &mut decoder, - true, - None, - )?; - - assert_ne!(piece, ""); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_to_piece_decodes_tool_call_variant(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut decoder = encoding_rs::UTF_8.new_decoder(); - let tokens = model.str_to_token("hi", AddBos::Never, ParseSpecialTokens::Always)?; - - let piece = - model.token_to_piece(&SampledToken::ToolCall(tokens[0]), &mut decoder, true, None)?; - - assert_ne!(piece, ""); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_to_piece_decodes_undeterminable_variant(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut decoder = encoding_rs::UTF_8.new_decoder(); - let tokens = model.str_to_token("hi", AddBos::Never, ParseSpecialTokens::Always)?; - - let piece = model.token_to_piece( - &SampledToken::Undeterminable(tokens[0]), - &mut decoder, - true, - None, - )?; - - assert_ne!(piece, ""); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn tokens_iterator_produces_valid_entries(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut count = 0; - - for (token, _piece_result) in model.tokens(false) { - assert!(token.0 >= 0); - count += 1; - - if count >= 100 { - break; - } - } - - assert_eq!(count, 100); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn n_vocab_matches_tokens_iterator_count(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let n_vocab = model.n_vocab(); - let count = model.tokens(false).count(); - - assert_eq!(count, usize::try_from(n_vocab)?); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128 -)] -fn debug_format_includes_struct_name_and_model_field(fixture: &LlamaFixture<'_>) -> Result<()> { - let formatted = format!("{:?}", fixture.model); - - assert!(formatted.contains("LlamaModel")); - assert!(formatted.contains("model")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn deepseek_r1_8b_adds_a_bos_token(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(fixture.model.adds_bos_token()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_does_not_add_a_bos_token(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(!fixture.model.adds_bos_token()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_to_piece_bytes_rejects_tokens_outside_the_vocabulary( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let n_vocab = model.n_vocab(); - - assert_eq!( - [LlamaToken::new(-1), LlamaToken::new(n_vocab)] - .map(|token| model.token_to_piece_bytes(token, 8, true, None)), - [ - Err(TokenToStringError::TokenOutsideVocabulary( - TokenOutsideVocabulary { - token: LlamaToken::new(-1), - n_vocab, - } - )), - Err(TokenToStringError::TokenOutsideVocabulary( - TokenOutsideVocabulary { - token: LlamaToken::new(n_vocab), - n_vocab, - } - )), - ] - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_attr_rejects_tokens_outside_the_vocabulary(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let n_vocab = model.n_vocab(); - - assert_eq!( - model.token_attr(LlamaToken::new(n_vocab)).err(), - Some(TokenAttrError::TokenOutsideVocabulary( - TokenOutsideVocabulary { - token: LlamaToken::new(n_vocab), - n_vocab, - } - )) - ); - - Ok(()) -} diff --git a/llama-cpp-bindings-tests/tests/model_loading_errors.rs b/llama-cpp-bindings-tests/tests/model_loading_errors.rs index e8b151a70..d6ab6b8db 100644 --- a/llama-cpp-bindings-tests/tests/model_loading_errors.rs +++ b/llama-cpp-bindings-tests/tests/model_loading_errors.rs @@ -17,7 +17,7 @@ use llama_cpp_test_harness::llama_test; n_batch = 128, n_ubatch = 64, )] -fn load_model_with_invalid_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { +fn load_model_with_invalid_path_returns_error(fixture: &LlamaFixture<'_>) { let model_params = LlamaModelParams::default(); let result = LlamaModel::load_from_file(fixture.backend, "/nonexistent/model.gguf", &model_params); @@ -26,7 +26,6 @@ fn load_model_with_invalid_path_returns_error(fixture: &LlamaFixture<'_>) -> Res result.unwrap_err(), LlamaModelLoadError::FileNotFound(path) if path == Path::new("/nonexistent/model.gguf"), )); - Ok(()) } #[llama_test( @@ -62,9 +61,7 @@ fn load_model_with_invalid_file_content_returns_unloadable_or_reported( n_batch = 128, n_ubatch = 64, )] -fn load_model_with_non_utf8_path_returns_path_to_str_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { +fn load_model_with_non_utf8_path_returns_path_to_str_error(fixture: &LlamaFixture<'_>) { use std::ffi::OsStr; use std::os::unix::ffi::OsStrExt; @@ -77,7 +74,6 @@ fn load_model_with_non_utf8_path_returns_path_to_str_error( result.unwrap_err(), LlamaModelLoadError::PathToStrError(path) if path == non_utf8_path.to_path_buf() )); - Ok(()) } #[llama_test( @@ -88,7 +84,7 @@ fn load_model_with_non_utf8_path_returns_path_to_str_error( n_batch = 128, n_ubatch = 64, )] -fn lora_adapter_init_with_invalid_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { +fn lora_adapter_init_with_invalid_path_returns_error(fixture: &LlamaFixture<'_>) { let result = fixture .model .lora_adapter_init("/nonexistent/path/lora.gguf"); @@ -96,7 +92,6 @@ fn lora_adapter_init_with_invalid_path_returns_error(fixture: &LlamaFixture<'_>) result.unwrap_err(), LlamaLoraAdapterInitError::FileNotFound(PathBuf::from("/nonexistent/path/lora.gguf")) ); - Ok(()) } #[llama_test( @@ -128,7 +123,7 @@ fn lora_adapter_init_with_invalid_gguf_returns_unloadable( n_batch = 128, n_ubatch = 64, )] -fn lora_adapter_init_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { +fn lora_adapter_init_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) { use std::ffi::OsStr; use std::os::unix::ffi::OsStrExt; @@ -139,5 +134,4 @@ fn lora_adapter_init_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_> result.unwrap_err(), LlamaLoraAdapterInitError::PathToStrError(non_utf8_path.to_path_buf()) ); - Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/chat_protocol.rs b/llama-cpp-bindings-tests/tests/model_metadata.rs similarity index 55% rename from llama-cpp-bindings-tests/tests/chat_protocol.rs rename to llama-cpp-bindings-tests/tests/model_metadata.rs index 0c4f13722..be2c9884c 100644 --- a/llama-cpp-bindings-tests/tests/chat_protocol.rs +++ b/llama-cpp-bindings-tests/tests/model_metadata.rs @@ -1,12 +1,4 @@ use anyhow::Result; -use anyhow::bail; -use llama_cpp_bindings::ChatMessageParseOutcome; -use llama_cpp_bindings::ChatMessageParser; -use llama_cpp_bindings::ChatTemplateError; -use llama_cpp_bindings::ParseChatMessageError; -use llama_cpp_bindings::model::LlamaChatMessage; -use llama_cpp_bindings_tests::build_user_prompt_with_media_marker::build_user_prompt_with_media_marker; -use llama_cpp_bindings_types::ParsedToolCall; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -42,10 +34,8 @@ use llama_cpp_test_harness::llama_test; n_batch = 128, n_ubatch = 64, )] -fn chat_template_returns_non_empty(fixture: &LlamaFixture<'_>) -> Result<()> { - let template = fixture.model.chat_template(None); - assert!(template.is_ok()); - Ok(()) +fn meta_count_returns_positive(fixture: &LlamaFixture<'_>) { + assert!(fixture.model.meta_count() > 0); } #[llama_test( @@ -80,54 +70,9 @@ fn chat_template_returns_non_empty(fixture: &LlamaFixture<'_>) -> Result<()> { n_batch = 128, n_ubatch = 64, )] -#[llama_test( - model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace( - "unsloth/Ministral-3-14B-Reasoning-2512-GGUF", - "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf" - ), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn apply_chat_template_produces_prompt(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let template = model.chat_template(None)?; - let message = LlamaChatMessage::new("user".to_string(), "hello".to_string())?; - let prompt = model.apply_chat_template(&template, &[message], true, true)?; - - assert!( - prompt.contains("hello"), - "the model's built-in chat template must render the user message content through the \ - engine; got: {prompt:?}" - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 64, - n_batch = 64, - n_ubatch = 64, -)] -fn build_user_prompt_surfaces_message_construction_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = build_user_prompt_with_media_marker(fixture.model, "describe\0this"); - - assert!( - result.is_err(), - "an interior null byte in the question must surface a message construction error" - ); +fn meta_key_by_index_returns_valid_key(fixture: &LlamaFixture<'_>) -> Result<()> { + let key = fixture.model.meta_key_by_index(0)?; + assert_ne!(key, ""); Ok(()) } @@ -163,17 +108,9 @@ fn build_user_prompt_surfaces_message_construction_error(fixture: &LlamaFixture< n_batch = 128, n_ubatch = 64, )] -fn apply_chat_template_renders_long_messages(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let template = model.chat_template(None)?; - let long_content = "a".repeat(2000); - let message = LlamaChatMessage::new("user".to_string(), long_content.clone())?; - let prompt = model.apply_chat_template(&template, &[message], true, true)?; - - assert!( - prompt.contains(&long_content), - "a long user message must be rendered in full by the engine without truncation" - ); +fn meta_val_str_by_index_returns_valid_value(fixture: &LlamaFixture<'_>) -> Result<()> { + let value = fixture.model.meta_val_str_by_index(0)?; + assert_ne!(value, ""); Ok(()) } @@ -209,12 +146,9 @@ fn apply_chat_template_renders_long_messages(fixture: &LlamaFixture<'_>) -> Resu n_batch = 128, n_ubatch = 64, )] -fn chat_template_with_nonexistent_name_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = fixture - .model - .chat_template(Some("nonexistent_template_name_xyz")); - assert_eq!(result.unwrap_err(), ChatTemplateError::MissingTemplate); - Ok(()) +fn meta_key_by_index_out_of_range_returns_error(fixture: &LlamaFixture<'_>) { + let result = fixture.model.meta_key_by_index(999_999); + assert!(result.is_err()); } #[llama_test( @@ -249,17 +183,9 @@ fn chat_template_with_nonexistent_name_returns_error(fixture: &LlamaFixture<'_>) n_batch = 128, n_ubatch = 64, )] -fn parses_pure_content_response(fixture: &LlamaFixture<'_>) -> Result<()> { - let outcome = ChatMessageParser::new(fixture.model, "[]")?.parse("hello world", false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!("expected Recognized for plain content; got Unrecognized"); - }; - assert_eq!(parsed.tool_calls, Vec::::new()); - assert!(!parsed.is_empty()); - assert!(parsed.content.contains("hello world")); - - Ok(()) +fn meta_val_str_by_index_out_of_range_returns_error(fixture: &LlamaFixture<'_>) { + let result = fixture.model.meta_val_str_by_index(999_999); + assert!(result.is_err()); } #[llama_test( @@ -294,20 +220,11 @@ fn parses_pure_content_response(fixture: &LlamaFixture<'_>) -> Result<()> { n_batch = 128, n_ubatch = 64, )] -fn parses_reasoning_section_into_reasoning_content(fixture: &LlamaFixture<'_>) -> Result<()> { - let input = "step one, step two\n\nactual response"; - let outcome = ChatMessageParser::new(fixture.model, "[]")?.parse(input, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!("expected Recognized for reasoning section; got Unrecognized"); - }; - assert!( - parsed.reasoning_content.contains("step") || parsed.content.contains("step"), - "neither content nor reasoning contains 'step'; content={:?} reasoning={:?}", - parsed.content, - parsed.reasoning_content - ); - +fn meta_val_str_returns_value_for_known_key(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let first_key = model.meta_key_by_index(0)?; + let value = model.meta_val_str(&first_key)?; + assert_ne!(value, ""); Ok(()) } @@ -343,65 +260,34 @@ fn parses_reasoning_section_into_reasoning_content(fixture: &LlamaFixture<'_>) - n_batch = 128, n_ubatch = 64, )] -fn parses_empty_input_yields_empty_message(fixture: &LlamaFixture<'_>) -> Result<()> { - let outcome = ChatMessageParser::new(fixture.model, "[]")?.parse("", false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!("expected Recognized for empty input; got Unrecognized"); - }; - assert_eq!(parsed.tool_calls, Vec::::new()); +fn meta_val_str_with_long_value_triggers_buffer_resize(fixture: &LlamaFixture<'_>) { + let model = fixture.model; + let count = model.meta_count(); - Ok(()) + for index in 0..count { + let key = model.meta_key_by_index(index); + let value = model.meta_val_str_by_index(index); + assert!(key.is_ok()); + assert!(value.is_ok()); + } } #[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, load_mode = Mmap, n_ctx = 512, n_batch = 128, n_ubatch = 64, )] -fn parses_with_input_null_byte_reports_the_input_as_the_source( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let result = ChatMessageParser::new(fixture.model, "[]")?.parse("hello\0world", false); - - let Err(ParseChatMessageError::InputContainsNulByte(nul_error)) = result else { - bail!("a NUL byte in the message must be reported against the message"); - }; - - assert_eq!(nul_error.nul_position(), 5); - - Ok(()) -} - #[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, load_mode = Mmap, n_ctx = 512, n_batch = 128, n_ubatch = 64, )] -fn chat_message_parser_for_a_tool_missing_its_function_name_fails_to_build( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let result = ChatMessageParser::new( - fixture.model, - r#"[{"type":"function","function":{"description":"reports the weather"}}]"#, - ); - - assert_eq!( - result.err(), - Some(ParseChatMessageError::ToolsParserBuildFailed { - message: "[json.exception.out_of_range.403] key 'name' not found".to_owned(), - }) - ); - - Ok(()) -} - #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -410,18 +296,15 @@ fn chat_message_parser_for_a_tool_missing_its_function_name_fails_to_build( n_batch = 128, n_ubatch = 64, )] -fn qwen35_chat_message_parser_parses_every_message_it_is_given( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let parser = ChatMessageParser::new(fixture.model, "[]")?; - - for message in ["the first answer", "the second answer"] { - let ChatMessageParseOutcome::Recognized(parsed) = parser.parse(message, false)? else { - bail!("expected {message:?} to be recognized as content"); - }; - - assert_eq!(parsed.content, message); - } - - Ok(()) +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn meta_val_str_with_null_byte_in_key_returns_error(fixture: &LlamaFixture<'_>) { + let result = fixture.model.meta_val_str("key\0with_null"); + assert!(result.is_err()); } diff --git a/llama-cpp-bindings-tests/tests/multimodal_audio.rs b/llama-cpp-bindings-tests/tests/multimodal_audio.rs index 371c9744d..27b903c16 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_audio.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_audio.rs @@ -1,6 +1,10 @@ use anyhow::Context; use anyhow::Result; use llama_cpp_bindings::BareJsonToolCalls; +use llama_cpp_bindings::ChatAssistantTurn; +use llama_cpp_bindings::ChatSession; +use llama_cpp_bindings::ChatSessionOptions; +use llama_cpp_bindings::ChatToolChoice; use llama_cpp_bindings::EvalMultimodalChunksParams; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; @@ -10,6 +14,7 @@ use llama_cpp_bindings::mtmd::MtmdBitmap; use llama_cpp_bindings::mtmd::MtmdCapability; use llama_cpp_bindings::mtmd::MtmdInputText; use llama_cpp_bindings::mtmd::mtmd_default_marker; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; @@ -38,14 +43,24 @@ fn transcribe_audio(fixture: &LlamaFixture<'_>, audio_file_name: &str) -> Result ); let marker = mtmd_default_marker()?; - let template = model.chat_template(None)?; let messages = [ LlamaChatMessage::new("system".to_string(), TRANSCRIBE_SYSTEM_PROMPT.to_string())?, LlamaChatMessage::new("user".to_string(), marker.to_owned())?, ]; + let session = ChatSession::new( + model, + &messages, + "[]", + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::New, + enable_thinking: true, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + )?; let input_text = MtmdInputText { - text: model.apply_chat_template(&template, &messages, true, true)?, - add_special: false, + text: session.prompt()?, + add_special: true, parse_special: true, }; @@ -67,7 +82,8 @@ fn transcribe_audio(fixture: &LlamaFixture<'_>, audio_file_name: &str) -> Result "tokenization should produce at least one chunk" ); - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; + let mut classifier = + SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); let n_past = classifier .eval_multimodal_chunks( &chunks, @@ -111,7 +127,7 @@ fn transcribe_audio(fixture: &LlamaFixture<'_>, audio_file_name: &str) -> Result } .run()?; - Ok(outcome.generated_raw.to_lowercase()) + Ok(outcome.generated.text().to_lowercase()) } fn assert_audio_transcription_contains( @@ -165,7 +181,7 @@ fn audio_reaches_the_model_and_produces_a_response(fixture: &LlamaFixture<'_>) - n_ubatch = 512, mmproj_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "mmproj-F16.gguf"), )] -fn audio_mmproj_reports_audio_support(fixture: &LlamaFixture<'_>) -> Result<()> { +fn audio_mmproj_reports_audio_support(fixture: &LlamaFixture<'_>) { let mtmd_ctx = fixture .mtmd_context .expect("mmproj_file declared in attribute"); @@ -180,8 +196,6 @@ fn audio_mmproj_reports_audio_support(fixture: &LlamaFixture<'_>) -> Result<()> mtmd_ctx.get_audio_sample_rate().is_some(), "an audio-capable mmproj must report a required sample rate" ); - - Ok(()) } #[llama_test( diff --git a/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs b/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs new file mode 100644 index 000000000..071ae6e84 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs @@ -0,0 +1,67 @@ +use anyhow::Result; +use llama_cpp_bindings::mtmd::MtmdBitmap; +use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn from_buffer_creates_bitmap_from_image_bytes(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + let fixtures = fixtures_dir(); + let image_path = fixtures.join("llamas.jpg"); + let image_bytes = std::fs::read(&image_path)?; + let bitmap = MtmdBitmap::from_buffer(mtmd_ctx, &image_bytes)?; + + assert!(bitmap.nx() > 0); + assert!(bitmap.ny() > 0); + assert!(!bitmap.is_audio()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn from_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_>) { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let result = MtmdBitmap::from_file(mtmd_ctx, "path\0null"); + + assert!(result.is_err()); +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs new file mode 100644 index 000000000..c2d2bb8d9 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs @@ -0,0 +1,47 @@ +use anyhow::Result; +use anyhow::anyhow; +use llama_cpp_bindings::mtmd::MtmdBitmap; +use llama_cpp_bindings::mtmd::MtmdInputText; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn copy_creates_owned_duplicate(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let image_data = vec![128u8; 64 * 64 * 3]; + let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; + let input_text = MtmdInputText { + text: "Hello <__media__>".to_string(), + add_special: true, + parse_special: true, + }; + let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + let first_chunk = chunks + .get(0) + .ok_or_else(|| anyhow!("missing first chunk"))?; + let copied = first_chunk.copy()?; + + assert!(copied.owned); + assert_eq!(copied.n_tokens(), first_chunk.n_tokens()); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs new file mode 100644 index 000000000..003ee0aa4 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs @@ -0,0 +1,51 @@ +use anyhow::Result; +use anyhow::anyhow; +use llama_cpp_bindings::mtmd::MtmdBitmap; +use llama_cpp_bindings::mtmd::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::MtmdInputText; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn encode_chunk_succeeds_for_image_chunk(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let image_data = vec![128u8; 64 * 64 * 3]; + let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; + let input_text = MtmdInputText { + text: "Describe: <__media__>".to_string(), + add_special: true, + parse_special: true, + }; + let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + + for chunk_index in 0..chunks.len() { + let chunk = chunks + .get(chunk_index) + .ok_or_else(|| anyhow!("missing chunk at index {chunk_index}"))?; + if chunk.chunk_type() == Ok(MtmdInputChunkType::Image) { + let result = mtmd_ctx.encode_chunk(&chunk); + assert!(result.is_ok()); + return Ok(()); + } + } + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs new file mode 100644 index 000000000..7e9a9915b --- /dev/null +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs @@ -0,0 +1,678 @@ +use anyhow::Context; +use anyhow::Result; +use anyhow::anyhow; +use anyhow::bail; +use llama_cpp_bindings::EvalMultimodalChunksParams; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::error::EvalMultimodalChunksError; +use llama_cpp_bindings::error::sequence_id_out_of_range::SequenceIdOutOfRange; +use llama_cpp_bindings::mtmd::FittedMtmdInputChunk; +use llama_cpp_bindings::mtmd::FittedMtmdInputChunks; +use llama_cpp_bindings::mtmd::MtmdBitmap; +use llama_cpp_bindings::mtmd::MtmdContext; +use llama_cpp_bindings::mtmd::MtmdEvalError; +use llama_cpp_bindings::mtmd::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::MtmdInputChunks; +use llama_cpp_bindings::mtmd::MtmdInputText; +use llama_cpp_bindings::mtmd::NonCausalChunkMicroBatchMismatch; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; +use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; +use llama_cpp_bindings_tests::media_question_chat_session::media_question_chat_session; +use llama_cpp_bindings_tests::prompt_question::PROMPT_QUESTION; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +use crate::tokenize_synthetic::tokenize_synthetic; + +fn eval_synthetic_bitmap(fixture: &LlamaFixture<'_>, width: u32, height: u32) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let image_data = vec![128u8; (width as usize) * (height as usize) * 3]; + let bitmap = MtmdBitmap::from_image_data(width, height, &image_data)?; + let input_text = MtmdInputText { + text: "Describe: <__media__>".to_string(), + add_special: true, + parse_special: true, + }; + let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + let n_positions = chunks.total_positions(); + let required_n_ctx = u32::try_from(n_positions + 256)?; + if fixture.context_params.n_ctx < required_n_ctx { + bail!( + "fixture n_ctx ({}) below required ({}) for {}x{} image", + fixture.context_params.n_ctx, + required_n_ctx, + width, + height, + ); + } + + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let n_batch = i32::try_from(llama_ctx.n_batch())?; + chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, false)?; + Ok(()) +} + +fn tokenize_question_about_llamas( + fixture: &LlamaFixture<'_>, + mtmd_ctx: &MtmdContext, +) -> Result { + let image_path = fixtures_dir().join("llamas.jpg"); + let image_path_str = image_path + .to_str() + .with_context(|| "image path is not valid UTF-8")?; + let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; + + Ok(mtmd_ctx.tokenize( + MtmdInputText { + text: media_question_chat_session(fixture.model, PROMPT_QUESTION)?.prompt()?, + add_special: true, + parse_special: true, + }, + &[&bitmap], + )?) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 64, + n_batch = 64, + n_ubatch = 32, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 64, + n_batch = 64, + n_ubatch = 32, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_chunks_returns_batch_size_exceeds_context_limit_for_huge_batch( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let chunks = MtmdInputChunks::new()?; + let huge_batch = i32::try_from(llama_ctx.n_batch() + 1)?; + + let result = chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, huge_batch, false); + + assert_eq!( + result, + Err(MtmdEvalError::BatchSizeExceedsContextLimit { + requested: huge_batch, + context_max: llama_ctx.n_batch(), + }) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_chunks_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_synthetic(fixture, "Describe: <__media__>")?; + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + assert_eq!( + chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, 0, true), + Err(MtmdEvalError::NonPositiveBatchSize { requested: 0 }) + ); + assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn fitting_no_chunks_leaves_nothing_to_evaluate(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let fitted_chunks = + FittedMtmdInputChunks::fit(MtmdInputChunks::new()?, mtmd_ctx, llama_ctx.n_ubatch())?; + + assert!(fitted_chunks.is_empty()); + assert!(FittedMtmdInputChunk::at(&fitted_chunks, 0).is_none()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_single_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let fitted_chunks = FittedMtmdInputChunks::fit( + tokenize_synthetic(fixture, "Describe: <__media__>")?, + mtmd_ctx, + llama_ctx.n_ubatch(), + )?; + let first_chunk = + FittedMtmdInputChunk::at(&fitted_chunks, 0).context("tokenization produced no chunks")?; + + assert_eq!( + first_chunk.eval_single(mtmd_ctx, &mut llama_ctx, 0, 0, 0, true), + Err(MtmdEvalError::NonPositiveBatchSize { requested: 0 }) + ); + assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_single_rejects_a_batch_larger_than_the_context_batch_before_evaluating( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let mut llama_ctx = fixture.build_context()?; + let fitted_chunks = FittedMtmdInputChunks::fit( + tokenize_synthetic(fixture, "Describe: <__media__>")?, + mtmd_ctx, + llama_ctx.n_ubatch(), + )?; + let first_chunk = + FittedMtmdInputChunk::at(&fitted_chunks, 0).context("tokenization produced no chunks")?; + let context_max = llama_ctx.n_batch(); + let oversized_batch = i32::try_from(context_max + 1)?; + + assert_eq!( + first_chunk.eval_single(mtmd_ctx, &mut llama_ctx, 0, 0, oversized_batch, true), + Err(MtmdEvalError::BatchSizeExceedsContextLimit { + requested: oversized_batch, + context_max, + }) + ); + assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_chunks_rejects_a_sequence_the_context_does_not_have_instead_of_aborting( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let mut llama_ctx = fixture.build_context()?; + let chunks = tokenize_synthetic(fixture, "Describe: <__media__>")?; + let n_seq_max = llama_ctx.n_seq_max(); + let missing_sequence = i32::try_from(n_seq_max)?; + let n_batch = i32::try_from(llama_ctx.n_batch())?; + + assert_eq!( + chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, missing_sequence, n_batch, true), + Err(MtmdEvalError::SequenceIdOutOfRange(SequenceIdOutOfRange { + seq_id: missing_sequence, + n_seq_max, + })) + ); + assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_single_rejects_a_micro_batch_smaller_than_the_fitted_one_before_evaluating( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let fitted_micro_batch_tokens = llama_ctx.n_ubatch(); + let fitted_chunks = FittedMtmdInputChunks::fit( + tokenize_synthetic(fixture, "Describe: <__media__>")?, + mtmd_ctx, + fitted_micro_batch_tokens, + )?; + let first_chunk = + FittedMtmdInputChunk::at(&fitted_chunks, 0).context("tokenization produced no chunks")?; + let smaller_micro_batch_tokens = fitted_micro_batch_tokens - 1; + + assert_eq!( + first_chunk.eval_single( + mtmd_ctx, + &mut llama_ctx, + 0, + 0, + i32::try_from(smaller_micro_batch_tokens)?, + true + ), + Err(MtmdEvalError::EvaluatedMicroBatchSmallerThanFitted { + evaluated_micro_batch_tokens: smaller_micro_batch_tokens, + fitted_micro_batch_tokens, + }) + ); + assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn classifier_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_synthetic(fixture, "Describe: <__media__>")?; + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let mut classifier = SampledTokenClassifier::undeterminable(fixture.model); + + assert!(matches!( + classifier.eval_multimodal_chunks( + &chunks, + mtmd_ctx, + &mut llama_ctx, + EvalMultimodalChunksParams { + start_position: 0, + seq_id: 0, + n_batch: 0, + logits_last: true, + }, + ), + Err(EvalMultimodalChunksError::EvalFailed( + MtmdEvalError::NonPositiveBatchSize { requested: 0 } + )) + )); + assert_eq!(classifier.usage().prompt_tokens, 0); + assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_chunks_succeeds_for_a_standard_image(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + let fixtures = fixtures_dir(); + let image_path = fixtures.join("llamas.jpg"); + let image_path_str = image_path + .to_str() + .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; + let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; + let input_text = MtmdInputText { + text: "What is in this image? <__media__>".to_string(), + add_special: true, + parse_special: true, + }; + let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + let n_positions = chunks.total_positions(); + let required_n_ctx = u32::try_from(n_positions + 256)?; + assert!( + fixture.context_params.n_ctx >= required_n_ctx, + "fixture n_ctx ({}) below required ({}); update the attribute literal", + fixture.context_params.n_ctx, + required_n_ctx, + ); + + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let n_batch = i32::try_from(llama_ctx.n_batch())?; + let result = chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, false); + + assert!(result.is_ok()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_chunks_succeeds_for_varied_image_dimensions(fixture: &LlamaFixture<'_>) { + let test_dimensions: [(u32, u32); 4] = [(224, 224), (512, 512), (100, 500), (337, 421)]; + + for (width, height) in test_dimensions { + let result = eval_synthetic_bitmap(fixture, width, height); + assert!( + result.is_ok(), + "dimension {width}x{height} should succeed: {result:?}" + ); + } +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_chunks_supports_extreme_dimensions(fixture: &LlamaFixture<'_>) -> Result<()> { + let extreme_dimensions: [(u32, u32); 6] = [ + (2, 2), + (7, 13), + (3, 1000), + (1000, 3), + (1920, 1080), + (4096, 4096), + ]; + + for (width, height) in extreme_dimensions { + eval_synthetic_bitmap(fixture, width, height) + .with_context(|| format!("dimension {width}x{height} should succeed"))?; + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 2048, + n_ubatch = 128, + mmproj_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "mmproj-F16.gguf"), +)] +fn gemma3_eval_chunks_rejects_a_non_causal_image_larger_than_the_micro_batch_before_evaluating( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; + let image_tokens = ChunkTokenBreakdown::from_chunks(&chunks)?.image; + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let n_batch = i32::try_from(llama_ctx.n_batch())?; + let micro_batch_tokens = llama_ctx.n_ubatch(); + + assert_eq!( + chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, true), + Err(MtmdEvalError::NonCausalChunkExceedsMicroBatch( + NonCausalChunkMicroBatchMismatch { + chunk_tokens: usize::try_from(image_tokens)?, + micro_batch_tokens, + } + )) + ); + assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 2048, + n_ubatch = 128, + mmproj_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "mmproj-F16.gguf"), +)] +fn gemma3_fitting_rejects_a_non_causal_image_larger_than_the_micro_batch( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; + let image_chunk = (0..chunks.len()) + .filter_map(|chunk_index| chunks.get(chunk_index)) + .find(|chunk| chunk.chunk_type() == Ok(MtmdInputChunkType::Image)) + .context("the prompt must contain an image chunk")?; + let llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + assert_eq!( + image_chunk.fit_to_micro_batch(mtmd_ctx, llama_ctx.n_ubatch()), + Err(MtmdEvalError::NonCausalChunkExceedsMicroBatch( + NonCausalChunkMicroBatchMismatch { + chunk_tokens: image_chunk.n_tokens(), + micro_batch_tokens: llama_ctx.n_ubatch(), + } + )) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 2048, + n_ubatch = 128, + mmproj_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "mmproj-F16.gguf"), +)] +fn gemma3_classifier_leaves_the_prompt_untouched_when_a_non_causal_image_exceeds_the_micro_batch( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let mut classifier = SampledTokenClassifier::undeterminable(fixture.model); + + let n_batch = i32::try_from(llama_ctx.n_batch())?; + let evaluation = classifier.eval_multimodal_chunks( + &chunks, + mtmd_ctx, + &mut llama_ctx, + EvalMultimodalChunksParams { + start_position: 0, + seq_id: 0, + n_batch, + logits_last: true, + }, + ); + + assert!(matches!( + evaluation, + Err(EvalMultimodalChunksError::EvalFailed( + MtmdEvalError::NonCausalChunkExceedsMicroBatch(NonCausalChunkMicroBatchMismatch { + micro_batch_tokens, + .. + }) + )) if micro_batch_tokens == llama_ctx.n_ubatch() + )); + assert_eq!(classifier.usage().prompt_tokens, 0); + assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs new file mode 100644 index 000000000..b5e3518ef --- /dev/null +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs @@ -0,0 +1,317 @@ +use anyhow::Result; +use anyhow::anyhow; +use llama_cpp_bindings::mtmd::MtmdBitmap; +use llama_cpp_bindings::mtmd::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::MtmdInputText; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +use crate::tokenize_synthetic::tokenize_synthetic; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn text_chunk_has_text_type(fixture: &LlamaFixture<'_>) -> Result<()> { + let chunks = tokenize_synthetic(fixture, "Hello world <__media__>")?; + let first_chunk = chunks + .get(0) + .ok_or_else(|| anyhow!("missing first chunk"))?; + assert_eq!(first_chunk.chunk_type()?, MtmdInputChunkType::Text); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn text_chunk_returns_text_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { + let chunks = tokenize_synthetic(fixture, "Hello world <__media__>")?; + let first_chunk = chunks + .get(0) + .ok_or_else(|| anyhow!("missing first chunk"))?; + let tokens = first_chunk + .text_tokens_of(first_chunk.chunk_type()?) + .ok_or_else(|| anyhow!("the first chunk of a text prompt must carry tokens"))?; + + assert_ne!(tokens, Vec::::new()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn chunk_n_tokens_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { + let chunks = tokenize_synthetic(fixture, "Hello world <__media__>")?; + let first_chunk = chunks + .get(0) + .ok_or_else(|| anyhow!("missing first chunk"))?; + assert!(first_chunk.n_tokens() > 0); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn chunk_n_positions_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { + let chunks = tokenize_synthetic(fixture, "Hello world <__media__>")?; + let first_chunk = chunks + .get(0) + .ok_or_else(|| anyhow!("missing first chunk"))?; + assert!(first_chunk.n_positions() > 0); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn text_chunk_id_returns_none(fixture: &LlamaFixture<'_>) -> Result<()> { + let chunks = tokenize_synthetic(fixture, "Hello <__media__>")?; + let first_chunk = chunks + .get(0) + .ok_or_else(|| anyhow!("missing first chunk"))?; + assert_eq!(first_chunk.chunk_type()?, MtmdInputChunkType::Text); + assert!(first_chunk.id().is_none()); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn image_chunk_returns_none_for_text_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { + let chunks = tokenize_synthetic(fixture, "Hello <__media__>")?; + for chunk_index in 0..chunks.len() { + let chunk = chunks + .get(chunk_index) + .ok_or_else(|| anyhow!("missing chunk at index {chunk_index}"))?; + if chunk.chunk_type()? == MtmdInputChunkType::Image { + assert_eq!(chunk.text_tokens_of(MtmdInputChunkType::Image), None); + + return Ok(()); + } + } + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn image_chunk_id_returns_some(fixture: &LlamaFixture<'_>) -> Result<()> { + let chunks = tokenize_synthetic(fixture, "Hello <__media__>")?; + for chunk_index in 0..chunks.len() { + let chunk = chunks + .get(chunk_index) + .ok_or_else(|| anyhow!("missing chunk at index {chunk_index}"))?; + if chunk.chunk_type() == Ok(MtmdInputChunkType::Image) { + assert!(chunk.id().is_some()); + return Ok(()); + } + } + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn tokenizing_text_with_an_image_produces_chunks(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let image_data = vec![128u8; 64 * 64 * 3]; + let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; + let input_text = MtmdInputText { + text: "Describe this image: <__media__>".to_string(), + add_special: true, + parse_special: true, + }; + let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + + assert!(!chunks.is_empty()); + assert!(chunks.total_tokens() > 0); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn tokenize_bitmap_count_mismatch_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let input_text = MtmdInputText { + text: "No media markers here".to_string(), + add_special: true, + parse_special: true, + }; + let image_data = vec![128u8; 64 * 64 * 3]; + let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; + let result = mtmd_ctx.tokenize(input_text, &[&bitmap]); + assert!(result.is_err()); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn tokenize_with_null_byte_in_text_returns_error(fixture: &LlamaFixture<'_>) { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let input_text = MtmdInputText { + text: "text\0null".to_string(), + add_special: true, + parse_special: true, + }; + let result = mtmd_ctx.tokenize(input_text, &[]); + assert!(result.is_err()); +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs index 3172fd413..d6591edac 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs @@ -1,6 +1,10 @@ use anyhow::Context; use anyhow::Result; use llama_cpp_bindings::BareJsonToolCalls; +use llama_cpp_bindings::ChatAssistantTurn; +use llama_cpp_bindings::ChatSession; +use llama_cpp_bindings::ChatSessionOptions; +use llama_cpp_bindings::ChatToolChoice; use llama_cpp_bindings::EvalMultimodalChunksParams; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; @@ -12,6 +16,7 @@ use llama_cpp_bindings::mtmd::MtmdCapability; use llama_cpp_bindings::mtmd::MtmdContext; use llama_cpp_bindings::mtmd::MtmdInputText; use llama_cpp_bindings::mtmd::mtmd_default_marker; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; @@ -25,13 +30,22 @@ const MAX_GENERATED_TOKENS: i32 = 512; const DESCRIBE_INSTRUCTION: &str = "Describe the animal shown in the image, then write the exact words spoken in the audio."; -fn build_describe_image_and_audio_prompt(model: &LlamaModel) -> Result { +fn describe_image_and_audio_chat_session(model: &LlamaModel) -> Result> { let marker = mtmd_default_marker()?; - let template = model.chat_template(None)?; let user_content = format!("Image: {marker}\nAudio: {marker}\n{DESCRIBE_INSTRUCTION}"); let messages = [LlamaChatMessage::new("user".to_string(), user_content)?]; - Ok(model.apply_chat_template(&template, &messages, true, false)?) + Ok(ChatSession::new( + model, + &messages, + "[]", + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::New, + enable_thinking: false, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + )?) } fn load_fixture_bitmap(mtmd_ctx: &MtmdContext, file_name: &str) -> Result { @@ -75,9 +89,10 @@ fn image_and_audio_together(fixture: &LlamaFixture<'_>) -> Result<()> { "orange_cat.wav must decode as audio" ); + let session = describe_image_and_audio_chat_session(model)?; let input_text = MtmdInputText { - text: build_describe_image_and_audio_prompt(model)?, - add_special: false, + text: session.prompt()?, + add_special: true, parse_special: true, }; @@ -111,7 +126,8 @@ fn image_and_audio_together(fixture: &LlamaFixture<'_>) -> Result<()> { .with_context(|| "unable to create llama context")?; let n_batch = i32::try_from(context.n_batch())?; - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; + let mut classifier = + SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); let n_past = classifier .eval_multimodal_chunks( &chunks, @@ -145,7 +161,7 @@ fn image_and_audio_together(fixture: &LlamaFixture<'_>) -> Result<()> { } .run()?; - let description = outcome.generated_raw.to_lowercase(); + let description = outcome.generated.text().to_lowercase(); assert!( !description.is_empty(), "model should generate a description from combined image and audio input" diff --git a/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs b/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs new file mode 100644 index 000000000..140d1a77b --- /dev/null +++ b/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs @@ -0,0 +1,141 @@ +use anyhow::Result; +use anyhow::anyhow; +use anyhow::bail; +use llama_cpp_bindings::mtmd::MtmdBitmap; +use llama_cpp_bindings::mtmd::MtmdCapability; +use llama_cpp_bindings::mtmd::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::MtmdInputText; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn decode_use_non_causal_returns_bool_for_image_chunk(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let image_data = vec![128u8; 64 * 64 * 3]; + let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; + let input_text = MtmdInputText { + text: "Describe: <__media__>".to_string(), + add_special: true, + parse_special: true, + }; + let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + for chunk_index in 0..chunks.len() { + let chunk = chunks + .get(chunk_index) + .ok_or_else(|| anyhow!("missing chunk at index {chunk_index}"))?; + if chunk.chunk_type() == Ok(MtmdInputChunkType::Image) { + let value = mtmd_ctx.decode_use_non_causal(&chunk); + let printed = format!("{value:?}"); + assert!( + !printed.is_empty(), + "decode_use_non_causal must return a Debug-printable bool" + ); + return Ok(()); + } + } + bail!("tokenization should produce at least one Image chunk"); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn qwen_mmproj_reports_vision_only_chat_capabilities(fixture: &LlamaFixture<'_>) { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + assert_eq!( + mtmd_ctx.capabilities(fixture.model), + MtmdCapability::Chat | MtmdCapability::Vision | MtmdCapability::TextGeneration + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn decode_use_mrope_is_true_for_qwen_vision(fixture: &LlamaFixture<'_>) { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + assert!( + mtmd_ctx.decode_use_mrope(), + "Qwen 3.5 / 3.6 mmproj uses mrope; decode_use_mrope must return true" + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn get_audio_sample_rate_is_none_for_vision_only_mmproj(fixture: &LlamaFixture<'_>) { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + assert!( + mtmd_ctx.get_audio_sample_rate().is_none(), + "Qwen 3.5 / 3.6 mmproj has no audio; get_audio_sample_rate must return None" + ); +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs b/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs new file mode 100644 index 000000000..38ec07a55 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs @@ -0,0 +1,29 @@ +use llama_cpp_bindings::mtmd::MtmdContext; +use llama_cpp_bindings::mtmd::MtmdContextParams; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn init_from_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_>) { + let mtmd_params = MtmdContextParams::default(); + let result = MtmdContext::init_from_file("path\0null", fixture.model, &mtmd_params); + + assert!(result.is_err()); +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs b/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs index 303883059..78665cce4 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs @@ -161,7 +161,7 @@ fn parse_special_applies_to_each_text_part_on_its_own(fixture: &LlamaFixture<'_> n_ubatch = 64, mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), )] -fn a_text_part_containing_a_nul_byte_is_rejected(fixture: &LlamaFixture<'_>) -> Result<()> { +fn a_text_part_containing_a_nul_byte_is_rejected(fixture: &LlamaFixture<'_>) { let mtmd_ctx = fixture .mtmd_context .expect("mmproj_file declared in attribute"); @@ -175,6 +175,4 @@ fn a_text_part_containing_a_nul_byte_is_rejected(fixture: &LlamaFixture<'_>) -> .err() .map(MtmdTokenizeError::CStringError) ); - - Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs new file mode 100644 index 000000000..b420d9451 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs @@ -0,0 +1,418 @@ +use anyhow::Result; +use anyhow::anyhow; +use anyhow::bail; +use llama_cpp_bindings::BareJsonToolCalls; +use llama_cpp_bindings::ChatAssistantTurn; +use llama_cpp_bindings::ChatSession; +use llama_cpp_bindings::ChatSessionOptions; +use llama_cpp_bindings::ChatToolChoice; +use llama_cpp_bindings::EvalMultimodalChunksParams; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::LlamaChatMessage; +use llama_cpp_bindings::mtmd::MtmdBitmap; +use llama_cpp_bindings::mtmd::MtmdInputText; +use llama_cpp_bindings::mtmd::mtmd_default_marker; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; +use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; +use llama_cpp_bindings_tests::media_question_chat_session::media_question_chat_session; +use llama_cpp_bindings_tests::prompt_question::PROMPT_QUESTION; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "mmproj-F16.gguf"), +)] +fn gemma4_classifier_emits_reasoning_for_multimodal_thinking_prompt( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + const MAX_GENERATED_TOKENS: i32 = 200; + + let model = fixture.model; + let backend = fixture.backend; + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + let mut context = LlamaContext::from_model( + model, + backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let image_path = fixtures_dir().join("llamas.jpg"); + let image_path_str = image_path + .to_str() + .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; + let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; + + let session = media_question_chat_session(model, PROMPT_QUESTION)?; + let input_text = MtmdInputText { + text: session.prompt()?, + add_special: true, + parse_special: true, + }; + + let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + + let mut classifier = + SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let n_past = classifier.eval_multimodal_chunks( + &chunks, + mtmd_ctx, + &mut context, + EvalMultimodalChunksParams { + start_position: 0, + seq_id: 0, + n_batch: 512, + logits_last: true, + }, + )?; + + let mut sampler = LlamaSampler::chain_simple([ + LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, + LlamaSampler::top_k(40)?, + LlamaSampler::top_p(0.9, 1)?, + LlamaSampler::min_p(0.05, 1)?, + LlamaSampler::temp(0.7)?, + LlamaSampler::dist(0x00C0_FFEE)?, + ])?; + + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + let outcome = ClassifySampleLoop { + classifier: &mut classifier, + sampler: &mut sampler, + context: &mut context, + batch: &mut batch, + initial_position: n_past, + max_generated_tokens: MAX_GENERATED_TOKENS, + } + .run()?; + + let usage = classifier.usage(); + + if outcome.observed_reasoning == 0 { + bail!( + "Gemma 4 multimodal + thinking: classifier must emit at least one Reasoning token \ + when the prompt opens a `<|channel>thought` block; outcome={outcome:?}" + ); + } + if usage.reasoning_tokens == 0 { + bail!( + "Gemma 4 multimodal + thinking: usage.reasoning_tokens must be non-zero; usage={usage:?}" + ); + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "mmproj-F16.gguf"), +)] +fn mistral3_classifier_emits_reasoning_for_multimodal_thinking_prompt( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + const MAX_GENERATED_TOKENS: i32 = 512; + + let model = fixture.model; + let backend = fixture.backend; + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + let mut context = LlamaContext::from_model( + model, + backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let image_path = fixtures_dir().join("llamas.jpg"); + let image_path_str = image_path + .to_str() + .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; + let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; + + let marker = mtmd_default_marker()?; + let messages = [ + LlamaChatMessage::new( + "system".to_owned(), + "# HOW YOU SHOULD THINK AND ANSWER\n\n\ + First draft your thinking process (inner monologue) until you arrive at a response. \ + Format your response using Markdown, and use LaTeX for any mathematical equations. \ + Write both your thoughts and the response in the same language as the input.\n\n\ + Your thinking process must follow the template below:\ + [THINK]Your thoughts or/and draft, like working through an exercise on scratch paper. \ + Be as casual and as long as you want until you are confident to generate the response \ + to the user.[/THINK]Here, provide a self-contained response." + .to_owned(), + )?, + LlamaChatMessage::new("user".to_owned(), format!("{marker}{PROMPT_QUESTION}"))?, + ]; + let session = ChatSession::new( + model, + &messages, + "[]", + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::New, + enable_thinking: true, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + )?; + let input_text = MtmdInputText { + text: session.prompt()?, + add_special: true, + parse_special: true, + }; + + let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + + let mut classifier = + SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let n_past = classifier.eval_multimodal_chunks( + &chunks, + mtmd_ctx, + &mut context, + EvalMultimodalChunksParams { + start_position: 0, + seq_id: 0, + n_batch: 512, + logits_last: true, + }, + )?; + + let mut sampler = LlamaSampler::greedy()?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + let outcome = ClassifySampleLoop { + classifier: &mut classifier, + sampler: &mut sampler, + context: &mut context, + batch: &mut batch, + initial_position: n_past, + max_generated_tokens: MAX_GENERATED_TOKENS, + } + .run()?; + + let usage = classifier.usage(); + + if outcome.observed_reasoning == 0 { + bail!( + "Mistral 3 multimodal + thinking: classifier must emit at least one Reasoning token \ + when the model opens a `[THINK]` block; outcome={outcome:?}" + ); + } + if usage.reasoning_tokens == 0 { + bail!( + "Mistral 3 multimodal + thinking: usage.reasoning_tokens must be non-zero; usage={usage:?}" + ); + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn qwen35_classifier_emits_reasoning_for_multimodal_thinking_prompt( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + const MAX_GENERATED_TOKENS: i32 = 200; + + let model = fixture.model; + let backend = fixture.backend; + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + let mut context = LlamaContext::from_model( + model, + backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let image_path = fixtures_dir().join("llamas.jpg"); + let image_path_str = image_path + .to_str() + .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; + let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; + + let session = media_question_chat_session(model, PROMPT_QUESTION)?; + let input_text = MtmdInputText { + text: session.prompt()?, + add_special: true, + parse_special: true, + }; + + let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + + let mut classifier = + SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let n_past = classifier.eval_multimodal_chunks( + &chunks, + mtmd_ctx, + &mut context, + EvalMultimodalChunksParams { + start_position: 0, + seq_id: 0, + n_batch: 512, + logits_last: true, + }, + )?; + + let mut sampler = LlamaSampler::chain_simple([ + LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, + LlamaSampler::top_k(40)?, + LlamaSampler::top_p(0.9, 1)?, + LlamaSampler::min_p(0.05, 1)?, + LlamaSampler::temp(0.7)?, + LlamaSampler::dist(0x00C0_FFEE)?, + ])?; + + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + let outcome = ClassifySampleLoop { + classifier: &mut classifier, + sampler: &mut sampler, + context: &mut context, + batch: &mut batch, + initial_position: n_past, + max_generated_tokens: MAX_GENERATED_TOKENS, + } + .run()?; + + let usage = classifier.usage(); + + if outcome.observed_reasoning == 0 { + bail!( + "Qwen 3.5 multimodal + thinking: classifier must emit at least one Reasoning token \ + when the prompt opens a `` block; outcome={outcome:?}" + ); + } + if usage.reasoning_tokens == 0 { + bail!( + "Qwen 3.5 multimodal + thinking: usage.reasoning_tokens must be non-zero; usage={usage:?}" + ); + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn qwen36_classifier_emits_reasoning_for_multimodal_thinking_prompt( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + const MAX_GENERATED_TOKENS: i32 = 200; + + let model = fixture.model; + let backend = fixture.backend; + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + let mut context = LlamaContext::from_model( + model, + backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let image_path = fixtures_dir().join("llamas.jpg"); + let image_path_str = image_path + .to_str() + .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; + let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; + + let session = media_question_chat_session(model, PROMPT_QUESTION)?; + let input_text = MtmdInputText { + text: session.prompt()?, + add_special: true, + parse_special: true, + }; + + let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + + let mut classifier = + SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let n_past = classifier.eval_multimodal_chunks( + &chunks, + mtmd_ctx, + &mut context, + EvalMultimodalChunksParams { + start_position: 0, + seq_id: 0, + n_batch: 512, + logits_last: true, + }, + )?; + + let mut sampler = LlamaSampler::chain_simple([ + LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, + LlamaSampler::top_k(40)?, + LlamaSampler::top_p(0.9, 1)?, + LlamaSampler::min_p(0.05, 1)?, + LlamaSampler::temp(0.7)?, + LlamaSampler::dist(0x00C0_FFEE)?, + ])?; + + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + let outcome = ClassifySampleLoop { + classifier: &mut classifier, + sampler: &mut sampler, + context: &mut context, + batch: &mut batch, + initial_position: n_past, + max_generated_tokens: MAX_GENERATED_TOKENS, + } + .run()?; + + let usage = classifier.usage(); + + if outcome.observed_reasoning == 0 { + bail!( + "Qwen 3.6 multimodal + thinking: classifier must emit at least one Reasoning token; outcome={outcome:?}" + ); + } + if usage.reasoning_tokens == 0 { + bail!( + "Qwen 3.6 multimodal + thinking: usage.reasoning_tokens must be non-zero; usage={usage:?}" + ); + } + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs b/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs new file mode 100644 index 000000000..e49d364b1 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs @@ -0,0 +1,301 @@ +use anyhow::Result; +use anyhow::anyhow; +use anyhow::bail; +use llama_cpp_bindings::EvalMultimodalChunksParams; +use llama_cpp_bindings::TokenUsage; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::mtmd::FittedMtmdInputChunk; +use llama_cpp_bindings::mtmd::FittedMtmdInputChunks; +use llama_cpp_bindings::mtmd::MtmdBitmap; +use llama_cpp_bindings::mtmd::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::MtmdInputText; +use llama_cpp_bindings::mtmd::mtmd_default_marker; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; +use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; +use llama_cpp_bindings_tests::prompt_question::PROMPT_QUESTION; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +fn build_multimodal_chunks_and_eval_into_usage( + fixture: &LlamaFixture<'_>, +) -> Result<(TokenUsage, ChunkTokenBreakdown)> { + let model = fixture.model; + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + let image_path = fixtures_dir().join("llamas.jpg"); + let image_path_str = image_path + .to_str() + .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; + let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; + + let marker = mtmd_default_marker()?; + let prompt = format!("{marker}{PROMPT_QUESTION}"); + + let input_text = MtmdInputText { + text: prompt, + add_special: false, + parse_special: true, + }; + + let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + let expected = ChunkTokenBreakdown::from_chunks(&chunks)?; + + let context_params = (*fixture.context_params).into_llama_context_params(); + let mut context = LlamaContext::from_model(model, fixture.backend, context_params)?; + + let mut classifier = SampledTokenClassifier::undeterminable(model); + classifier.eval_multimodal_chunks( + &chunks, + mtmd_ctx, + &mut context, + EvalMultimodalChunksParams { + start_position: 0, + seq_id: 0, + n_batch: 512, + logits_last: true, + }, + )?; + + Ok((classifier.into_usage(), expected)) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn prompt_tokens_match_text_chunk_total(fixture: &LlamaFixture<'_>) -> Result<()> { + let (usage, expected) = build_multimodal_chunks_and_eval_into_usage(fixture)?; + + if usage.prompt_tokens != expected.text { + bail!( + "prompt_tokens must equal sum of text-chunk n_tokens; expected {}, got {}", + expected.text, + usage.prompt_tokens + ); + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn input_image_tokens_match_image_chunk_total(fixture: &LlamaFixture<'_>) -> Result<()> { + let (usage, expected) = build_multimodal_chunks_and_eval_into_usage(fixture)?; + + if usage.input_image_tokens != expected.image { + bail!( + "input_image_tokens must equal sum of image-chunk n_tokens; expected {}, got {}", + expected.image, + usage.input_image_tokens + ); + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn input_audio_tokens_are_zero_for_image_only_input(fixture: &LlamaFixture<'_>) -> Result<()> { + let (usage, expected) = build_multimodal_chunks_and_eval_into_usage(fixture)?; + + if expected.audio != 0 { + bail!( + "fixture invariant: image-only multimodal input should produce zero audio chunk tokens, got {}", + expected.audio + ); + } + if usage.input_audio_tokens != 0 { + bail!( + "input_audio_tokens must be zero when no audio chunks are evaluated; got {}", + usage.input_audio_tokens + ); + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn completion_tokens_are_zero_after_eval_before_generation( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let (usage, _expected) = build_multimodal_chunks_and_eval_into_usage(fixture)?; + + if usage.completion_tokens() != 0 { + bail!( + "completion_tokens must be zero immediately after eval (no generation has occurred); got {}", + usage.completion_tokens() + ); + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn text_chunk_records_prompt_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + let input_text = MtmdInputText { + text: "hello world".to_owned(), + add_special: false, + parse_special: false, + }; + let llama_ctx = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let fitted_chunks = FittedMtmdInputChunks::fit( + mtmd_ctx.tokenize(input_text, &[])?, + mtmd_ctx, + llama_ctx.n_ubatch(), + )?; + + let text_chunk = (0..fitted_chunks.len()) + .filter_map(|index| FittedMtmdInputChunk::at(&fitted_chunks, index)) + .find(|chunk| chunk.chunk_type() == MtmdInputChunkType::Text) + .ok_or_else(|| anyhow!("text-only tokenization should produce at least one text chunk"))?; + + let n_tokens = u64::try_from(text_chunk.chunk().n_tokens())?; + + let mut classifier = SampledTokenClassifier::undeterminable(model); + + classifier.ingest_fitted_chunk(&text_chunk); + + let usage = classifier.usage(); + if usage.prompt_tokens != n_tokens { + bail!( + "text chunk must record n_tokens as prompt_tokens; expected {n_tokens}, got {}", + usage.prompt_tokens + ); + } + if usage.input_image_tokens != 0 { + bail!( + "text chunk must not bump input_image_tokens; got {}", + usage.input_image_tokens + ); + } + if usage.input_audio_tokens != 0 { + bail!( + "text chunk must not bump input_audio_tokens; got {}", + usage.input_audio_tokens + ); + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn image_chunk_records_input_image_tokens_only(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + let image_path = fixtures_dir().join("llamas.jpg"); + let image_path_str = image_path + .to_str() + .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; + let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; + + let marker = mtmd_default_marker()?; + let input_text = MtmdInputText { + text: marker.to_owned(), + add_special: false, + parse_special: true, + }; + let llama_ctx = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let fitted_chunks = FittedMtmdInputChunks::fit( + mtmd_ctx.tokenize(input_text, &[&bitmap])?, + mtmd_ctx, + llama_ctx.n_ubatch(), + )?; + + let image_chunk = (0..fitted_chunks.len()) + .filter_map(|index| FittedMtmdInputChunk::at(&fitted_chunks, index)) + .find(|chunk| chunk.chunk_type() == MtmdInputChunkType::Image) + .ok_or_else(|| anyhow!("multimodal tokenization should produce an image chunk"))?; + + let n_tokens = u64::try_from(image_chunk.chunk().n_tokens())?; + if n_tokens == 0 { + bail!("image chunk should report at least one token"); + } + + let mut classifier = SampledTokenClassifier::undeterminable(model); + + classifier.ingest_fitted_chunk(&image_chunk); + + let usage = classifier.usage(); + if usage.input_image_tokens != n_tokens { + bail!( + "image chunk must record n_tokens as input_image_tokens; expected {n_tokens}, got {}", + usage.input_image_tokens + ); + } + if usage.prompt_tokens != 0 { + bail!( + "image chunk must not bump prompt_tokens; got {}", + usage.prompt_tokens + ); + } + if usage.input_audio_tokens != 0 { + bail!( + "image chunk must not bump input_audio_tokens; got {}", + usage.input_audio_tokens + ); + } + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_vision.rs b/llama-cpp-bindings-tests/tests/multimodal_vision.rs deleted file mode 100644 index a4860f214..000000000 --- a/llama-cpp-bindings-tests/tests/multimodal_vision.rs +++ /dev/null @@ -1,2172 +0,0 @@ -use anyhow::Context; -use anyhow::Result; -use anyhow::anyhow; -use anyhow::bail; -use llama_cpp_bindings::BareJsonToolCalls; -use llama_cpp_bindings::EvalMultimodalChunksParams; -use llama_cpp_bindings::TokenUsage; -use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_bindings::context::context_limits::ContextLimits; -use llama_cpp_bindings::error::EvalMultimodalChunksError; -use llama_cpp_bindings::error::sequence_id_out_of_range::SequenceIdOutOfRange; -use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::mtmd::FittedMtmdInputChunk; -use llama_cpp_bindings::mtmd::FittedMtmdInputChunks; -use llama_cpp_bindings::mtmd::MtmdBitmap; -use llama_cpp_bindings::mtmd::MtmdCapability; -use llama_cpp_bindings::mtmd::MtmdContext; -use llama_cpp_bindings::mtmd::MtmdContextParams; -use llama_cpp_bindings::mtmd::MtmdEvalError; -use llama_cpp_bindings::mtmd::MtmdInputChunkType; -use llama_cpp_bindings::mtmd::MtmdInputChunks; -use llama_cpp_bindings::mtmd::MtmdInputText; -use llama_cpp_bindings::mtmd::NonCausalChunkMicroBatchMismatch; -use llama_cpp_bindings::mtmd::mtmd_default_marker; -use llama_cpp_bindings::sampling::LlamaSampler; -use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_bindings_tests::build_user_prompt_with_media_marker::build_user_prompt_with_media_marker; -use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; -use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; -use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn from_buffer_creates_bitmap_from_image_bytes(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let fixtures = fixtures_dir(); - let image_path = fixtures.join("llamas.jpg"); - let image_bytes = std::fs::read(&image_path)?; - let bitmap = MtmdBitmap::from_buffer(mtmd_ctx, &image_bytes)?; - - assert!(bitmap.nx() > 0); - assert!(bitmap.ny() > 0); - assert!(!bitmap.is_audio()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn from_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let result = MtmdBitmap::from_file(mtmd_ctx, "path\0null"); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn copy_creates_owned_duplicate(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let image_data = vec![128u8; 64 * 64 * 3]; - let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; - let input_text = MtmdInputText { - text: "Hello <__media__>".to_string(), - add_special: true, - parse_special: true, - }; - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - let first_chunk = chunks - .get(0) - .ok_or_else(|| anyhow!("missing first chunk"))?; - let copied = first_chunk.copy()?; - - assert!(copied.owned); - assert_eq!(copied.n_tokens(), first_chunk.n_tokens()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn encode_chunk_succeeds_for_image_chunk(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let image_data = vec![128u8; 64 * 64 * 3]; - let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; - let input_text = MtmdInputText { - text: "Describe: <__media__>".to_string(), - add_special: true, - parse_special: true, - }; - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - - for chunk_index in 0..chunks.len() { - let chunk = chunks - .get(chunk_index) - .ok_or_else(|| anyhow!("missing chunk at index {chunk_index}"))?; - if chunk.chunk_type() == Ok(MtmdInputChunkType::Image) { - let result = mtmd_ctx.encode_chunk(&chunk); - assert!(result.is_ok()); - return Ok(()); - } - } - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn decode_use_non_causal_returns_bool_for_image_chunk(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let image_data = vec![128u8; 64 * 64 * 3]; - let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; - let input_text = MtmdInputText { - text: "Describe: <__media__>".to_string(), - add_special: true, - parse_special: true, - }; - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - for chunk_index in 0..chunks.len() { - let chunk = chunks - .get(chunk_index) - .ok_or_else(|| anyhow!("missing chunk at index {chunk_index}"))?; - if chunk.chunk_type() == Ok(MtmdInputChunkType::Image) { - let value = mtmd_ctx.decode_use_non_causal(&chunk); - let printed = format!("{value:?}"); - assert!( - !printed.is_empty(), - "decode_use_non_causal must return a Debug-printable bool" - ); - return Ok(()); - } - } - bail!("tokenization should produce at least one Image chunk"); -} - -fn tokenize_synthetic(fixture: &LlamaFixture<'_>, prompt: &str) -> Result { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let image_data = vec![128u8; 64 * 64 * 3]; - let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; - let input_text = MtmdInputText { - text: prompt.to_owned(), - add_special: true, - parse_special: true, - }; - Ok(mtmd_ctx.tokenize(input_text, &[&bitmap])?) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn text_chunk_has_text_type(fixture: &LlamaFixture<'_>) -> Result<()> { - let chunks = tokenize_synthetic(fixture, "Hello world <__media__>")?; - let first_chunk = chunks - .get(0) - .ok_or_else(|| anyhow!("missing first chunk"))?; - assert_eq!(first_chunk.chunk_type()?, MtmdInputChunkType::Text); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn text_chunk_returns_text_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { - let chunks = tokenize_synthetic(fixture, "Hello world <__media__>")?; - let first_chunk = chunks - .get(0) - .ok_or_else(|| anyhow!("missing first chunk"))?; - let tokens = first_chunk - .text_tokens_of(first_chunk.chunk_type()?) - .ok_or_else(|| anyhow!("the first chunk of a text prompt must carry tokens"))?; - - assert_ne!(tokens, Vec::::new()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn chunk_n_tokens_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { - let chunks = tokenize_synthetic(fixture, "Hello world <__media__>")?; - let first_chunk = chunks - .get(0) - .ok_or_else(|| anyhow!("missing first chunk"))?; - assert!(first_chunk.n_tokens() > 0); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn chunk_n_positions_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { - let chunks = tokenize_synthetic(fixture, "Hello world <__media__>")?; - let first_chunk = chunks - .get(0) - .ok_or_else(|| anyhow!("missing first chunk"))?; - assert!(first_chunk.n_positions() > 0); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn text_chunk_id_returns_none(fixture: &LlamaFixture<'_>) -> Result<()> { - let chunks = tokenize_synthetic(fixture, "Hello <__media__>")?; - let first_chunk = chunks - .get(0) - .ok_or_else(|| anyhow!("missing first chunk"))?; - assert_eq!(first_chunk.chunk_type()?, MtmdInputChunkType::Text); - assert!(first_chunk.id().is_none()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn image_chunk_returns_none_for_text_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { - let chunks = tokenize_synthetic(fixture, "Hello <__media__>")?; - for chunk_index in 0..chunks.len() { - let chunk = chunks - .get(chunk_index) - .ok_or_else(|| anyhow!("missing chunk at index {chunk_index}"))?; - if chunk.chunk_type()? == MtmdInputChunkType::Image { - assert_eq!(chunk.text_tokens_of(MtmdInputChunkType::Image), None); - - return Ok(()); - } - } - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn image_chunk_id_returns_some(fixture: &LlamaFixture<'_>) -> Result<()> { - let chunks = tokenize_synthetic(fixture, "Hello <__media__>")?; - for chunk_index in 0..chunks.len() { - let chunk = chunks - .get(chunk_index) - .ok_or_else(|| anyhow!("missing chunk at index {chunk_index}"))?; - if chunk.chunk_type() == Ok(MtmdInputChunkType::Image) { - assert!(chunk.id().is_some()); - return Ok(()); - } - } - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn qwen_mmproj_reports_vision_only_chat_capabilities(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - assert_eq!( - mtmd_ctx.capabilities(fixture.model), - MtmdCapability::Chat | MtmdCapability::Vision | MtmdCapability::TextGeneration - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn init_from_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_params = MtmdContextParams::default(); - let result = MtmdContext::init_from_file("path\0null", fixture.model, &mtmd_params); - - assert!(result.is_err()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn decode_use_mrope_is_true_for_qwen_vision(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - assert!( - mtmd_ctx.decode_use_mrope(), - "Qwen 3.5 / 3.6 mmproj uses mrope; decode_use_mrope must return true" - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn get_audio_sample_rate_is_none_for_vision_only_mmproj(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - assert!( - mtmd_ctx.get_audio_sample_rate().is_none(), - "Qwen 3.5 / 3.6 mmproj has no audio; get_audio_sample_rate must return None" - ); - Ok(()) -} - -fn eval_synthetic_bitmap(fixture: &LlamaFixture<'_>, width: u32, height: u32) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let image_data = vec![128u8; (width as usize) * (height as usize) * 3]; - let bitmap = MtmdBitmap::from_image_data(width, height, &image_data)?; - let input_text = MtmdInputText { - text: "Describe: <__media__>".to_string(), - add_special: true, - parse_special: true, - }; - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - let n_positions = chunks.total_positions(); - let required_n_ctx = u32::try_from(n_positions + 256)?; - if fixture.context_params.n_ctx < required_n_ctx { - bail!( - "fixture n_ctx ({}) below required ({}) for {}x{} image", - fixture.context_params.n_ctx, - required_n_ctx, - width, - height, - ); - } - - let mut llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let n_batch = i32::try_from(llama_ctx.n_batch())?; - chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, false)?; - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 64, - n_batch = 64, - n_ubatch = 32, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 64, - n_batch = 64, - n_ubatch = 32, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn eval_chunks_returns_batch_size_exceeds_context_limit_for_huge_batch( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let mut llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let chunks = MtmdInputChunks::new()?; - let huge_batch = i32::try_from(llama_ctx.n_batch() + 1)?; - - let result = chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, huge_batch, false); - - assert_eq!( - result, - Err(MtmdEvalError::BatchSizeExceedsContextLimit { - requested: huge_batch, - context_max: llama_ctx.n_batch(), - }) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn eval_chunks_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let chunks = tokenize_synthetic(fixture, "Describe: <__media__>")?; - let mut llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - assert_eq!( - chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, 0, true), - Err(MtmdEvalError::NonPositiveBatchSize { requested: 0 }) - ); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn fitting_no_chunks_leaves_nothing_to_evaluate(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let fitted_chunks = - FittedMtmdInputChunks::fit(MtmdInputChunks::new()?, mtmd_ctx, llama_ctx.n_ubatch())?; - - assert!(fitted_chunks.is_empty()); - assert!(FittedMtmdInputChunk::at(&fitted_chunks, 0).is_none()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn eval_single_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let mut llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let fitted_chunks = FittedMtmdInputChunks::fit( - tokenize_synthetic(fixture, "Describe: <__media__>")?, - mtmd_ctx, - llama_ctx.n_ubatch(), - )?; - let first_chunk = - FittedMtmdInputChunk::at(&fitted_chunks, 0).context("tokenization produced no chunks")?; - - assert_eq!( - first_chunk.eval_single(mtmd_ctx, &mut llama_ctx, 0, 0, 0, true), - Err(MtmdEvalError::NonPositiveBatchSize { requested: 0 }) - ); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -fn eval_single_rejects_a_batch_larger_than_the_context_batch_before_evaluating( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let mut llama_ctx = fixture.build_context()?; - let fitted_chunks = FittedMtmdInputChunks::fit( - tokenize_synthetic(fixture, "Describe: <__media__>")?, - mtmd_ctx, - llama_ctx.n_ubatch(), - )?; - let first_chunk = - FittedMtmdInputChunk::at(&fitted_chunks, 0).context("tokenization produced no chunks")?; - let context_max = llama_ctx.n_batch(); - let oversized_batch = i32::try_from(context_max + 1)?; - - assert_eq!( - first_chunk.eval_single(mtmd_ctx, &mut llama_ctx, 0, 0, oversized_batch, true), - Err(MtmdEvalError::BatchSizeExceedsContextLimit { - requested: oversized_batch, - context_max, - }) - ); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -fn eval_chunks_rejects_a_sequence_the_context_does_not_have_instead_of_aborting( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let mut llama_ctx = fixture.build_context()?; - let chunks = tokenize_synthetic(fixture, "Describe: <__media__>")?; - let n_seq_max = llama_ctx.n_seq_max(); - let missing_sequence = i32::try_from(n_seq_max)?; - let n_batch = i32::try_from(llama_ctx.n_batch())?; - - assert_eq!( - chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, missing_sequence, n_batch, true), - Err(MtmdEvalError::SequenceIdOutOfRange(SequenceIdOutOfRange { - seq_id: missing_sequence, - n_seq_max, - })) - ); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn eval_single_rejects_a_micro_batch_smaller_than_the_fitted_one_before_evaluating( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let mut llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let fitted_micro_batch_tokens = llama_ctx.n_ubatch(); - let fitted_chunks = FittedMtmdInputChunks::fit( - tokenize_synthetic(fixture, "Describe: <__media__>")?, - mtmd_ctx, - fitted_micro_batch_tokens, - )?; - let first_chunk = - FittedMtmdInputChunk::at(&fitted_chunks, 0).context("tokenization produced no chunks")?; - let smaller_micro_batch_tokens = fitted_micro_batch_tokens - 1; - - assert_eq!( - first_chunk.eval_single( - mtmd_ctx, - &mut llama_ctx, - 0, - 0, - i32::try_from(smaller_micro_batch_tokens)?, - true - ), - Err(MtmdEvalError::EvaluatedMicroBatchSmallerThanFitted { - evaluated_micro_batch_tokens: smaller_micro_batch_tokens, - fitted_micro_batch_tokens, - }) - ); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn classifier_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let chunks = tokenize_synthetic(fixture, "Describe: <__media__>")?; - let mut llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let mut classifier = fixture - .model - .sampled_token_classifier(BareJsonToolCalls::Ignore)?; - - assert!(matches!( - classifier.eval_multimodal_chunks( - &chunks, - mtmd_ctx, - &mut llama_ctx, - EvalMultimodalChunksParams { - start_position: 0, - seq_id: 0, - n_batch: 0, - logits_last: true, - }, - ), - Err(EvalMultimodalChunksError::EvalFailed( - MtmdEvalError::NonPositiveBatchSize { requested: 0 } - )) - )); - assert_eq!(classifier.usage().prompt_tokens, 0); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn eval_chunks_with_standard_image(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let fixtures = fixtures_dir(); - let image_path = fixtures.join("llamas.jpg"); - let image_path_str = image_path - .to_str() - .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; - let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; - let input_text = MtmdInputText { - text: "What is in this image? <__media__>".to_string(), - add_special: true, - parse_special: true, - }; - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - let n_positions = chunks.total_positions(); - let required_n_ctx = u32::try_from(n_positions + 256)?; - assert!( - fixture.context_params.n_ctx >= required_n_ctx, - "fixture n_ctx ({}) below required ({}); update the attribute literal", - fixture.context_params.n_ctx, - required_n_ctx, - ); - - let mut llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let n_batch = i32::try_from(llama_ctx.n_batch())?; - let result = chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, false); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn eval_chunks_with_varied_dimensions(fixture: &LlamaFixture<'_>) -> Result<()> { - let test_dimensions: [(u32, u32); 4] = [(224, 224), (512, 512), (100, 500), (337, 421)]; - - for (width, height) in test_dimensions { - let result = eval_synthetic_bitmap(fixture, width, height); - assert!( - result.is_ok(), - "dimension {width}x{height} should succeed: {result:?}" - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn eval_chunks_supports_extreme_dimensions(fixture: &LlamaFixture<'_>) -> Result<()> { - let extreme_dimensions: [(u32, u32); 6] = [ - (2, 2), - (7, 13), - (3, 1000), - (1000, 3), - (1920, 1080), - (4096, 4096), - ]; - - for (width, height) in extreme_dimensions { - eval_synthetic_bitmap(fixture, width, height) - .with_context(|| format!("dimension {width}x{height} should succeed"))?; - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn tokenize_text_with_image(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let image_data = vec![128u8; 64 * 64 * 3]; - let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; - let input_text = MtmdInputText { - text: "Describe this image: <__media__>".to_string(), - add_special: true, - parse_special: true, - }; - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - - assert!(!chunks.is_empty()); - assert!(chunks.total_tokens() > 0); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn tokenize_bitmap_count_mismatch_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let input_text = MtmdInputText { - text: "No media markers here".to_string(), - add_special: true, - parse_special: true, - }; - let image_data = vec![128u8; 64 * 64 * 3]; - let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; - let result = mtmd_ctx.tokenize(input_text, &[&bitmap]); - assert!(result.is_err()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn tokenize_with_null_byte_in_text_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let input_text = MtmdInputText { - text: "text\0null".to_string(), - add_special: true, - parse_special: true, - }; - let result = mtmd_ctx.tokenize(input_text, &[]); - assert!(result.is_err()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4096, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -fn multimodal_vision_inference_produces_output(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let mut ctx = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - ) - .with_context(|| "unable to create llama context")?; - - assert!( - mtmd_ctx - .capabilities(model) - .contains(MtmdCapability::Vision), - "model should support vision input" - ); - - let image_path = fixtures_dir().join("llamas.jpg"); - let image_path_str = image_path - .to_str() - .with_context(|| "image path is not valid UTF-8")?; - let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str) - .with_context(|| "failed to load image from file")?; - - let formatted_prompt = - build_user_prompt_with_media_marker(model, "What animals do you see in this image?")?; - - let input_text = MtmdInputText { - text: formatted_prompt, - add_special: false, - parse_special: true, - }; - - let chunks = mtmd_ctx - .tokenize(input_text, &[&bitmap]) - .with_context(|| "failed to tokenize multimodal input")?; - - assert!( - !chunks.is_empty(), - "tokenization should produce at least one chunk" - ); - - let expected = ChunkTokenBreakdown::from_chunks(&chunks)?; - - eprintln!( - "tokenized into {} chunks, text {} image {} audio {}", - chunks.len(), - expected.text, - expected.image, - expected.audio - ); - - assert!( - expected.image > 0, - "vision input must produce at least one image chunk" - ); - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let n_past = classifier - .eval_multimodal_chunks( - &chunks, - mtmd_ctx, - &mut ctx, - EvalMultimodalChunksParams { - start_position: 0, - seq_id: 0, - n_batch: 512, - logits_last: true, - }, - ) - .with_context(|| "failed to evaluate chunks")?; - - eprintln!("evaluated chunks, n_past = {n_past}"); - - { - let usage = classifier.usage(); - assert_eq!(usage.prompt_tokens, expected.text); - assert_eq!(usage.input_image_tokens, expected.image); - assert_eq!(usage.input_audio_tokens, expected.audio); - } - - let mut sampler = LlamaSampler::greedy()?; - let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); - let totals = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut ctx, - batch: &mut batch, - initial_position: n_past, - max_generated_tokens: 512, - } - .run()?; - - eprintln!("generated text: {}", totals.generated_raw); - - assert!( - !totals.generated_raw.is_empty(), - "model should generate at least one token from image input" - ); - - let usage = classifier.into_usage(); - assert_eq!(usage.prompt_tokens, expected.text); - assert_eq!(usage.input_image_tokens, expected.image); - assert_eq!(usage.input_audio_tokens, expected.audio); - assert_eq!(usage.content_tokens, totals.observed_content); - assert_eq!(usage.reasoning_tokens, totals.observed_reasoning); - assert_eq!( - usage.completion_tokens(), - totals.observed_content + totals.observed_reasoning - ); - - Ok(()) -} - -const PROMPT_QUESTION: &str = "What animals do you see in this image?"; - -fn build_multimodal_chunks_and_eval_into_usage( - fixture: &LlamaFixture<'_>, -) -> Result<(TokenUsage, ChunkTokenBreakdown)> { - let model = fixture.model; - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let image_path = fixtures_dir().join("llamas.jpg"); - let image_path_str = image_path - .to_str() - .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; - let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; - - let marker = mtmd_default_marker()?; - let prompt = format!("{marker}{PROMPT_QUESTION}"); - - let input_text = MtmdInputText { - text: prompt, - add_special: false, - parse_special: true, - }; - - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - let expected = ChunkTokenBreakdown::from_chunks(&chunks)?; - - let context_params = (*fixture.context_params).into_llama_context_params(); - let mut context = LlamaContext::from_model(model, fixture.backend, context_params)?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - classifier.eval_multimodal_chunks( - &chunks, - mtmd_ctx, - &mut context, - EvalMultimodalChunksParams { - start_position: 0, - seq_id: 0, - n_batch: 512, - logits_last: true, - }, - )?; - - Ok((classifier.into_usage(), expected)) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4096, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -fn prompt_tokens_match_text_chunk_total(fixture: &LlamaFixture<'_>) -> Result<()> { - let (usage, expected) = build_multimodal_chunks_and_eval_into_usage(fixture)?; - - if usage.prompt_tokens != expected.text { - bail!( - "prompt_tokens must equal sum of text-chunk n_tokens; expected {}, got {}", - expected.text, - usage.prompt_tokens - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4096, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -fn input_image_tokens_match_image_chunk_total(fixture: &LlamaFixture<'_>) -> Result<()> { - let (usage, expected) = build_multimodal_chunks_and_eval_into_usage(fixture)?; - - if usage.input_image_tokens != expected.image { - bail!( - "input_image_tokens must equal sum of image-chunk n_tokens; expected {}, got {}", - expected.image, - usage.input_image_tokens - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4096, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -fn input_audio_tokens_are_zero_for_image_only_input(fixture: &LlamaFixture<'_>) -> Result<()> { - let (usage, expected) = build_multimodal_chunks_and_eval_into_usage(fixture)?; - - if expected.audio != 0 { - bail!( - "fixture invariant: image-only multimodal input should produce zero audio chunk tokens, got {}", - expected.audio - ); - } - if usage.input_audio_tokens != 0 { - bail!( - "input_audio_tokens must be zero when no audio chunks are evaluated; got {}", - usage.input_audio_tokens - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4096, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -fn completion_tokens_are_zero_after_eval_before_generation( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let (usage, _expected) = build_multimodal_chunks_and_eval_into_usage(fixture)?; - - if usage.completion_tokens() != 0 { - bail!( - "completion_tokens must be zero immediately after eval (no generation has occurred); got {}", - usage.completion_tokens() - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -fn text_chunk_records_prompt_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let input_text = MtmdInputText { - text: "hello world".to_owned(), - add_special: false, - parse_special: false, - }; - let llama_ctx = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let fitted_chunks = FittedMtmdInputChunks::fit( - mtmd_ctx.tokenize(input_text, &[])?, - mtmd_ctx, - llama_ctx.n_ubatch(), - )?; - - let text_chunk = (0..fitted_chunks.len()) - .filter_map(|index| FittedMtmdInputChunk::at(&fitted_chunks, index)) - .find(|chunk| chunk.chunk_type() == MtmdInputChunkType::Text) - .ok_or_else(|| anyhow!("text-only tokenization should produce at least one text chunk"))?; - - let n_tokens = u64::try_from(text_chunk.chunk().n_tokens())?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - - classifier.ingest_fitted_chunk(&text_chunk); - - let usage = classifier.usage(); - if usage.prompt_tokens != n_tokens { - bail!( - "text chunk must record n_tokens as prompt_tokens; expected {n_tokens}, got {}", - usage.prompt_tokens - ); - } - if usage.input_image_tokens != 0 { - bail!( - "text chunk must not bump input_image_tokens; got {}", - usage.input_image_tokens - ); - } - if usage.input_audio_tokens != 0 { - bail!( - "text chunk must not bump input_audio_tokens; got {}", - usage.input_audio_tokens - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -fn image_chunk_records_input_image_tokens_only(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let image_path = fixtures_dir().join("llamas.jpg"); - let image_path_str = image_path - .to_str() - .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; - let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; - - let marker = mtmd_default_marker()?; - let input_text = MtmdInputText { - text: marker.to_owned(), - add_special: false, - parse_special: true, - }; - let llama_ctx = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let fitted_chunks = FittedMtmdInputChunks::fit( - mtmd_ctx.tokenize(input_text, &[&bitmap])?, - mtmd_ctx, - llama_ctx.n_ubatch(), - )?; - - let image_chunk = (0..fitted_chunks.len()) - .filter_map(|index| FittedMtmdInputChunk::at(&fitted_chunks, index)) - .find(|chunk| chunk.chunk_type() == MtmdInputChunkType::Image) - .ok_or_else(|| anyhow!("multimodal tokenization should produce an image chunk"))?; - - let n_tokens = u64::try_from(image_chunk.chunk().n_tokens())?; - if n_tokens == 0 { - bail!("image chunk should report at least one token"); - } - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - - classifier.ingest_fitted_chunk(&image_chunk); - - let usage = classifier.usage(); - if usage.input_image_tokens != n_tokens { - bail!( - "image chunk must record n_tokens as input_image_tokens; expected {n_tokens}, got {}", - usage.input_image_tokens - ); - } - if usage.prompt_tokens != 0 { - bail!( - "image chunk must not bump prompt_tokens; got {}", - usage.prompt_tokens - ); - } - if usage.input_audio_tokens != 0 { - bail!( - "image chunk must not bump input_audio_tokens; got {}", - usage.input_audio_tokens - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -fn text_chunk_drives_marker_state_machine_to_reasoning(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let input_text = MtmdInputText { - text: "<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\n\n".to_owned(), - add_special: false, - parse_special: true, - }; - let llama_ctx = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let fitted_chunks = FittedMtmdInputChunks::fit( - mtmd_ctx.tokenize(input_text, &[])?, - mtmd_ctx, - llama_ctx.n_ubatch(), - )?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - - for index in 0..fitted_chunks.len() { - let chunk = FittedMtmdInputChunk::at(&fitted_chunks, index) - .ok_or_else(|| anyhow!("chunk index {index} must exist"))?; - classifier.ingest_fitted_chunk(&chunk); - } - - if classifier.current_section() != llama_cpp_bindings::SampledTokenSection::Reasoning { - bail!( - "text chunk replay must transition the classifier section to Reasoning when the \ - prompt opens a `` block; got {:?}", - classifier.current_section() - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "mmproj-F16.gguf"), -)] -fn gemma4_classifier_emits_reasoning_for_multimodal_thinking_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 200; - - let model = fixture.model; - let backend = fixture.backend; - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let image_path = fixtures_dir().join("llamas.jpg"); - let image_path_str = image_path - .to_str() - .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; - let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; - - let marker = mtmd_default_marker()?; - let prompt = format!( - "user\n{marker}What animals do you see in this image?\nmodel\n<|channel>thought\n" - ); - - let input_text = MtmdInputText { - text: prompt, - add_special: false, - parse_special: true, - }; - - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let n_past = classifier.eval_multimodal_chunks( - &chunks, - mtmd_ctx, - &mut context, - EvalMultimodalChunksParams { - start_position: 0, - seq_id: 0, - n_batch: 512, - logits_last: true, - }, - )?; - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position: n_past, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - - if outcome.observed_reasoning == 0 { - bail!( - "Gemma 4 multimodal + thinking: classifier must emit at least one Reasoning token \ - when the prompt opens a `<|channel>thought` block; outcome={outcome:?}" - ); - } - if usage.reasoning_tokens == 0 { - bail!( - "Gemma 4 multimodal + thinking: usage.reasoning_tokens must be non-zero; usage={usage:?}" - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4096, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "mmproj-F16.gguf"), -)] -fn mistral3_classifier_emits_reasoning_for_multimodal_thinking_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 512; - - let model = fixture.model; - let backend = fixture.backend; - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let image_path = fixtures_dir().join("llamas.jpg"); - let image_path_str = image_path - .to_str() - .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; - let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; - - let marker = mtmd_default_marker()?; - let prompt = format!( - "[SYSTEM_PROMPT]# HOW YOU SHOULD THINK AND ANSWER\n\n\ - First draft your thinking process (inner monologue) until you arrive at a response. \ - Format your response using Markdown, and use LaTeX for any mathematical equations. \ - Write both your thoughts and the response in the same language as the input.\n\n\ - Your thinking process must follow the template below:\ - [THINK]Your thoughts or/and draft, like working through an exercise on scratch paper. \ - Be as casual and as long as you want until you are confident to generate the response \ - to the user.[/THINK]Here, provide a self-contained response.[/SYSTEM_PROMPT]\ - [INST]{marker}What animals do you see in this image?[/INST]" - ); - - let input_text = MtmdInputText { - text: prompt, - add_special: true, - parse_special: true, - }; - - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let n_past = classifier.eval_multimodal_chunks( - &chunks, - mtmd_ctx, - &mut context, - EvalMultimodalChunksParams { - start_position: 0, - seq_id: 0, - n_batch: 512, - logits_last: true, - }, - )?; - - let mut sampler = LlamaSampler::greedy()?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position: n_past, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - - if outcome.observed_reasoning == 0 { - bail!( - "Mistral 3 multimodal + thinking: classifier must emit at least one Reasoning token \ - when the model opens a `[THINK]` block; outcome={outcome:?}" - ); - } - if usage.reasoning_tokens == 0 { - bail!( - "Mistral 3 multimodal + thinking: usage.reasoning_tokens must be non-zero; usage={usage:?}" - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4096, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4096, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn qwen35_classifier_emits_reasoning_for_multimodal_thinking_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 200; - - let model = fixture.model; - let backend = fixture.backend; - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let image_path = fixtures_dir().join("llamas.jpg"); - let image_path_str = image_path - .to_str() - .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; - let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; - - let marker = mtmd_default_marker()?; - let prompt = format!( - "<|im_start|>user\n{marker}What animals do you see in this image?<|im_end|>\n<|im_start|>assistant\n\n" - ); - - let input_text = MtmdInputText { - text: prompt, - add_special: false, - parse_special: true, - }; - - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let n_past = classifier.eval_multimodal_chunks( - &chunks, - mtmd_ctx, - &mut context, - EvalMultimodalChunksParams { - start_position: 0, - seq_id: 0, - n_batch: 512, - logits_last: true, - }, - )?; - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position: n_past, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - - if outcome.observed_reasoning == 0 { - bail!( - "Qwen 3.5 multimodal + thinking: classifier must emit at least one Reasoning token \ - when the prompt opens a `` block; outcome={outcome:?}" - ); - } - if usage.reasoning_tokens == 0 { - bail!( - "Qwen 3.5 multimodal + thinking: usage.reasoning_tokens must be non-zero; usage={usage:?}" - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn qwen36_classifier_emits_reasoning_for_multimodal_thinking_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 200; - - let model = fixture.model; - let backend = fixture.backend; - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let image_path = fixtures_dir().join("llamas.jpg"); - let image_path_str = image_path - .to_str() - .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; - let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; - - let marker = mtmd_default_marker()?; - let prompt = format!( - "<|im_start|>user\n{marker}What animals do you see in this image?<|im_end|>\n<|im_start|>assistant\n\n" - ); - - let input_text = MtmdInputText { - text: prompt, - add_special: false, - parse_special: true, - }; - - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let n_past = classifier.eval_multimodal_chunks( - &chunks, - mtmd_ctx, - &mut context, - EvalMultimodalChunksParams { - start_position: 0, - seq_id: 0, - n_batch: 512, - logits_last: true, - }, - )?; - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position: n_past, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - - if outcome.observed_reasoning == 0 { - bail!( - "Qwen 3.6 multimodal + thinking: classifier must emit at least one Reasoning token; outcome={outcome:?}" - ); - } - if usage.reasoning_tokens == 0 { - bail!( - "Qwen 3.6 multimodal + thinking: usage.reasoning_tokens must be non-zero; usage={usage:?}" - ); - } - - Ok(()) -} - -fn tokenize_question_about_llamas( - fixture: &LlamaFixture<'_>, - mtmd_ctx: &MtmdContext, -) -> Result { - let image_path = fixtures_dir().join("llamas.jpg"); - let image_path_str = image_path - .to_str() - .with_context(|| "image path is not valid UTF-8")?; - let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; - - Ok(mtmd_ctx.tokenize( - MtmdInputText { - text: build_user_prompt_with_media_marker(fixture.model, PROMPT_QUESTION)?, - add_special: false, - parse_special: true, - }, - &[&bitmap], - )?) -} - -#[llama_test( - model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 2048, - n_ubatch = 128, - mmproj_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "mmproj-F16.gguf"), -)] -fn gemma3_eval_chunks_rejects_a_non_causal_image_larger_than_the_micro_batch_before_evaluating( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; - let image_tokens = ChunkTokenBreakdown::from_chunks(&chunks)?.image; - let mut llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let n_batch = i32::try_from(llama_ctx.n_batch())?; - let micro_batch_tokens = llama_ctx.n_ubatch(); - - assert_eq!( - chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, true), - Err(MtmdEvalError::NonCausalChunkExceedsMicroBatch( - NonCausalChunkMicroBatchMismatch { - chunk_tokens: usize::try_from(image_tokens)?, - micro_batch_tokens, - } - )) - ); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 2048, - n_ubatch = 128, - mmproj_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "mmproj-F16.gguf"), -)] -fn gemma3_fitting_rejects_a_non_causal_image_larger_than_the_micro_batch( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; - let image_chunk = (0..chunks.len()) - .filter_map(|chunk_index| chunks.get(chunk_index)) - .find(|chunk| chunk.chunk_type() == Ok(MtmdInputChunkType::Image)) - .context("the prompt must contain an image chunk")?; - let llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - assert_eq!( - image_chunk.fit_to_micro_batch(mtmd_ctx, llama_ctx.n_ubatch()), - Err(MtmdEvalError::NonCausalChunkExceedsMicroBatch( - NonCausalChunkMicroBatchMismatch { - chunk_tokens: image_chunk.n_tokens(), - micro_batch_tokens: llama_ctx.n_ubatch(), - } - )) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 2048, - n_ubatch = 128, - mmproj_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "mmproj-F16.gguf"), -)] -fn gemma3_classifier_leaves_the_prompt_untouched_when_a_non_causal_image_exceeds_the_micro_batch( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; - let mut llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let mut classifier = fixture - .model - .sampled_token_classifier(BareJsonToolCalls::Ignore)?; - - let n_batch = i32::try_from(llama_ctx.n_batch())?; - let evaluation = classifier.eval_multimodal_chunks( - &chunks, - mtmd_ctx, - &mut llama_ctx, - EvalMultimodalChunksParams { - start_position: 0, - seq_id: 0, - n_batch, - logits_last: true, - }, - ); - - assert!(matches!( - evaluation, - Err(EvalMultimodalChunksError::EvalFailed( - MtmdEvalError::NonCausalChunkExceedsMicroBatch(NonCausalChunkMicroBatchMismatch { - micro_batch_tokens, - .. - }) - )) if micro_batch_tokens == llama_ctx.n_ubatch() - )); - assert_eq!(classifier.usage().prompt_tokens, 0); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); - - Ok(()) -} diff --git a/llama-cpp-bindings-tests/tests/multimodal_vision_inference.rs b/llama-cpp-bindings-tests/tests/multimodal_vision_inference.rs new file mode 100644 index 000000000..adc23bdcf --- /dev/null +++ b/llama-cpp-bindings-tests/tests/multimodal_vision_inference.rs @@ -0,0 +1,143 @@ +use anyhow::Context; +use anyhow::Result; +use llama_cpp_bindings::BareJsonToolCalls; +use llama_cpp_bindings::EvalMultimodalChunksParams; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::mtmd::MtmdBitmap; +use llama_cpp_bindings::mtmd::MtmdCapability; +use llama_cpp_bindings::mtmd::MtmdInputText; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; +use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; +use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; +use llama_cpp_bindings_tests::media_question_chat_session::media_question_chat_session; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn multimodal_vision_inference_produces_output(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + let mut ctx = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + ) + .with_context(|| "unable to create llama context")?; + + assert!( + mtmd_ctx + .capabilities(model) + .contains(MtmdCapability::Vision), + "model should support vision input" + ); + + let image_path = fixtures_dir().join("llamas.jpg"); + let image_path_str = image_path + .to_str() + .with_context(|| "image path is not valid UTF-8")?; + let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str) + .with_context(|| "failed to load image from file")?; + + let session = media_question_chat_session(model, "What animals do you see in this image?")?; + let input_text = MtmdInputText { + text: session.prompt()?, + add_special: true, + parse_special: true, + }; + + let chunks = mtmd_ctx + .tokenize(input_text, &[&bitmap]) + .with_context(|| "failed to tokenize multimodal input")?; + + assert!( + !chunks.is_empty(), + "tokenization should produce at least one chunk" + ); + + let expected = ChunkTokenBreakdown::from_chunks(&chunks)?; + + eprintln!( + "tokenized into {} chunks, text {} image {} audio {}", + chunks.len(), + expected.text, + expected.image, + expected.audio + ); + + assert!( + expected.image > 0, + "vision input must produce at least one image chunk" + ); + + let mut classifier = + SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let n_past = classifier + .eval_multimodal_chunks( + &chunks, + mtmd_ctx, + &mut ctx, + EvalMultimodalChunksParams { + start_position: 0, + seq_id: 0, + n_batch: 512, + logits_last: true, + }, + ) + .with_context(|| "failed to evaluate chunks")?; + + eprintln!("evaluated chunks, n_past = {n_past}"); + + { + let usage = classifier.usage(); + assert_eq!(usage.prompt_tokens, expected.text); + assert_eq!(usage.input_image_tokens, expected.image); + assert_eq!(usage.input_audio_tokens, expected.audio); + } + + let mut sampler = LlamaSampler::greedy()?; + let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); + let totals = ClassifySampleLoop { + classifier: &mut classifier, + sampler: &mut sampler, + context: &mut ctx, + batch: &mut batch, + initial_position: n_past, + max_generated_tokens: 512, + } + .run()?; + + eprintln!("generated text: {}", totals.generated.text()); + + assert!( + !totals.generated.text().is_empty(), + "model should generate at least one token from image input" + ); + + let usage = classifier.into_usage(); + assert_eq!(usage.prompt_tokens, expected.text); + assert_eq!(usage.input_image_tokens, expected.image); + assert_eq!(usage.input_audio_tokens, expected.audio); + assert_eq!(usage.content_tokens, totals.observed_content); + assert_eq!(usage.reasoning_tokens, totals.observed_reasoning); + assert_eq!( + usage.completion_tokens(), + totals.observed_content + totals.observed_reasoning + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/parameter_fitting.rs b/llama-cpp-bindings-tests/tests/parameter_fitting.rs new file mode 100644 index 000000000..13222ad3d --- /dev/null +++ b/llama-cpp-bindings-tests/tests/parameter_fitting.rs @@ -0,0 +1,126 @@ +use std::ffi::CString; +use std::num::NonZeroU32; +use std::pin::pin; + +use anyhow::Result; +use anyhow::anyhow; +use llama_cpp_bindings::context::params::LlamaContextParams; +use llama_cpp_bindings::max_devices; +use llama_cpp_bindings::model::params::LlamaModelParams; +use llama_cpp_bindings::model::params::fit_extra_model::FitExtraModel; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn fit_params_succeeds_with_test_model(fixture: &LlamaFixture<'_>) -> Result<()> { + let model_path_utf8 = fixture + .model_path + .to_str() + .ok_or_else(|| anyhow!("model path is not valid UTF-8"))?; + let model_path_c = CString::new(model_path_utf8)?; + + let mut params = pin!(LlamaModelParams::default()); + let mut context_params = LlamaContextParams::default(); + let mut margins = vec![0usize; max_devices()]; + + let result = params.as_mut().fit_params( + &model_path_c, + &mut context_params, + &mut margins, + 512, + None, + llama_cpp_bindings_sys::GGML_LOG_LEVEL_NONE, + ); + + let fit = result.map_err(|fit_error| anyhow!("fit_params failed: {fit_error:?}"))?; + assert!(fit.n_ctx > 0); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn fit_params_aligns_the_extra_model_context_with_the_fitted_one( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model_path_utf8 = fixture + .model_path + .to_str() + .ok_or_else(|| anyhow!("model path is not valid UTF-8"))?; + let model_path_c = CString::new(model_path_utf8)?; + + let mut params = pin!(LlamaModelParams::default()); + let mut context_params = LlamaContextParams::default(); + let mut margins = vec![0usize; max_devices()]; + + let unfittable_extra_n_ctx = NonZeroU32::MIN; + let mut extra_params = pin!(LlamaModelParams::default()); + let mut extra_context_params = + LlamaContextParams::default().with_n_ctx(Some(unfittable_extra_n_ctx)); + let mut extra = FitExtraModel { + model_path: &model_path_c, + model_params: extra_params.as_mut(), + context_params: &mut extra_context_params, + shares_model: true, + }; + + let result = params.as_mut().fit_params( + &model_path_c, + &mut context_params, + &mut margins, + 512, + Some(&mut extra), + llama_cpp_bindings_sys::GGML_LOG_LEVEL_NONE, + ); + + let fit = result.map_err(|fit_error| anyhow!("fit_params failed: {fit_error:?}"))?; + assert!(fit.n_ctx > 0); + assert_ne!( + extra_context_params.n_ctx(), + Some(unfittable_extra_n_ctx), + "the llama.cpp fit must overwrite the extra model's context size" + ); + assert_eq!( + extra_context_params.context_params.n_ctx, + context_params.context_params.n_ctx + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/performance_timings.rs b/llama-cpp-bindings-tests/tests/performance_timings.rs new file mode 100644 index 000000000..48613ca1b --- /dev/null +++ b/llama-cpp-bindings-tests/tests/performance_timings.rs @@ -0,0 +1,26 @@ +use anyhow::Result; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn timings_after_reset_report_a_non_negative_start_time(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + context.reset_timings(); + let timings = context.timings(); + assert!(timings.t_start_ms() >= 0.0); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs b/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs new file mode 100644 index 000000000..90c04803c --- /dev/null +++ b/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs @@ -0,0 +1,254 @@ +use anyhow::Context as _; +use anyhow::Result; +use llama_cpp_bindings::SampledTokenClassifier; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn feed_prompt_to_batch_increments_pending_prompt_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let mut classifier = SampledTokenClassifier::undeterminable(model); + let mut batch = LlamaBatch::new(ContextLimits { + n_batch: 8, + n_ubatch: 8, + n_seq_max: 1, + n_vocab: model.n_vocab(), + }); + + classifier.feed_prompt_to_batch( + &mut batch, + model.token_bos().context("model declares a BOS token")?, + 0, + &[0], + false, + )?; + classifier.feed_prompt_to_batch( + &mut batch, + model.token_eos().context("model declares an EOS token")?, + 1, + &[0], + false, + )?; + + assert_eq!(classifier.pending_prompt_tokens(), 2); + assert_eq!(batch.n_tokens(), 2); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn feed_prompt_sequence_to_batch_stages_all_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let mut classifier = SampledTokenClassifier::undeterminable(model); + let mut batch = LlamaBatch::new(ContextLimits { + n_batch: 8, + n_ubatch: 8, + n_seq_max: 1, + n_vocab: model.n_vocab(), + }); + + let tokens = vec![ + model.token_bos().context("model declares a BOS token")?, + model.token_eos().context("model declares an EOS token")?, + model.token_nl().context("model declares a newline token")?, + ]; + classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; + + assert_eq!(classifier.pending_prompt_tokens(), 3); + assert_eq!(batch.n_tokens(), 3); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn commit_prompt_tokens_promotes_pending_count_to_usage_and_clears( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let mut classifier = SampledTokenClassifier::undeterminable(model); + let mut batch = LlamaBatch::new(ContextLimits { + n_batch: 8, + n_ubatch: 8, + n_seq_max: 1, + n_vocab: model.n_vocab(), + }); + + classifier.feed_prompt_to_batch( + &mut batch, + model.token_bos().context("model declares a BOS token")?, + 0, + &[0], + false, + )?; + classifier.feed_prompt_to_batch( + &mut batch, + model.token_eos().context("model declares an EOS token")?, + 1, + &[0], + false, + )?; + + let promoted = classifier.commit_prompt_tokens(); + + assert_eq!(promoted, 2); + assert_eq!(classifier.pending_prompt_tokens(), 0); + assert_eq!(classifier.usage().prompt_tokens, 2); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn discard_pending_prompt_tokens_clears_count_without_recording_usage( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let mut classifier = SampledTokenClassifier::undeterminable(model); + let mut batch = LlamaBatch::new(ContextLimits { + n_batch: 8, + n_ubatch: 8, + n_seq_max: 1, + n_vocab: model.n_vocab(), + }); + + classifier.feed_prompt_to_batch( + &mut batch, + model.token_bos().context("model declares a BOS token")?, + 0, + &[0], + false, + )?; + + let discarded = classifier.discard_pending_prompt_tokens(); + + assert_eq!(discarded, 1); + assert_eq!(classifier.pending_prompt_tokens(), 0); + assert_eq!(classifier.usage().prompt_tokens, 0); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/reasoning_classification.rs b/llama-cpp-bindings-tests/tests/reasoning_classification.rs new file mode 100644 index 000000000..c4f516ee3 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/reasoning_classification.rs @@ -0,0 +1,309 @@ +use anyhow::Result; +use llama_cpp_bindings::ChatAssistantTurn; +use llama_cpp_bindings::ChatSession; +use llama_cpp_bindings::ChatSessionOptions; +use llama_cpp_bindings::ChatToolChoice; +use llama_cpp_bindings::model::LlamaChatMessage; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings_tests::chat_answer_generation::ChatAnswerGeneration; +use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; +use llama_cpp_bindings_tests::penalized_nucleus_sampler::penalized_nucleus_sampler; +use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +const ARITHMETIC_QUESTION: &str = "What is 2 + 2?"; +const SINGLE_WORD_INSTRUCTION: &str = "Reply with the single word: four. Do not explain."; +const MAX_ANSWER_TOKENS: i32 = 200; +const MAX_REASONING_ANSWER_TOKENS: i32 = 512; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn deepseek_r1_8b_answers_without_reasoning_after_its_reasoning_is_closed( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = closed_reasoning_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]")?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut penalized_nucleus_sampler(fixture.model)?, + max_generated_tokens: MAX_ANSWER_TOKENS, + } + .run()?; + + answer.assert_content_without_reasoning(&session) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn deepseek_r1_8b_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]", true)?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut penalized_nucleus_sampler(fixture.model)?, + max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, + } + .run()?; + + answer.assert_reasoning_matches_parse(&session) +} + +#[llama_test( + model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn gemma4_answers_without_reasoning_when_thinking_is_disabled( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, SINGLE_WORD_INSTRUCTION, "[]", false)?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut LlamaSampler::greedy()?, + max_generated_tokens: MAX_ANSWER_TOKENS, + } + .run()?; + + answer.assert_content_without_reasoning(&session) +} + +#[llama_test( + model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn gemma4_reasoning_streams_match_the_parsed_answer_when_reasoning_is_prefilled( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = ChatSession::new( + fixture.model, + &[LlamaChatMessage::new( + "user".to_owned(), + SINGLE_WORD_INSTRUCTION.to_owned(), + )?], + "[]", + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::ContinueReasoning( + "The user wants one word.".to_owned(), + ), + enable_thinking: true, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + )?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut LlamaSampler::greedy()?, + max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, + } + .run()?; + + answer.assert_reasoning_matches_parse(&session) +} + +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn glm47_answers_without_reasoning_when_thinking_is_disabled( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]", false)?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut penalized_nucleus_sampler(fixture.model)?, + max_generated_tokens: MAX_ANSWER_TOKENS, + } + .run()?; + + answer.assert_content_without_reasoning(&session) +} + +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn glm47_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]", true)?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut penalized_nucleus_sampler(fixture.model)?, + max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, + } + .run()?; + + answer.assert_reasoning_matches_parse(&session) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn mistral3_answers_without_reasoning_after_its_reasoning_is_closed( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = closed_reasoning_chat_session(fixture.model, SINGLE_WORD_INSTRUCTION, "[]")?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut LlamaSampler::greedy()?, + max_generated_tokens: MAX_ANSWER_TOKENS, + } + .run()?; + + answer.assert_content_without_reasoning(&session) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn mistral3_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, SINGLE_WORD_INSTRUCTION, "[]", true)?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut LlamaSampler::greedy()?, + max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, + } + .run()?; + + answer.assert_reasoning_matches_parse(&session) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn qwen35_answers_without_reasoning_when_thinking_is_disabled( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]", false)?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut penalized_nucleus_sampler(fixture.model)?, + max_generated_tokens: MAX_ANSWER_TOKENS, + } + .run()?; + + answer.assert_content_without_reasoning(&session) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn qwen35_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]", true)?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut penalized_nucleus_sampler(fixture.model)?, + max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, + } + .run()?; + + answer.assert_reasoning_matches_parse(&session) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn qwen36_answers_without_reasoning_when_thinking_is_disabled( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]", false)?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut penalized_nucleus_sampler(fixture.model)?, + max_generated_tokens: MAX_ANSWER_TOKENS, + } + .run()?; + + answer.assert_content_without_reasoning(&session) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn qwen36_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]", true)?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut penalized_nucleus_sampler(fixture.model)?, + max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, + } + .run()?; + + answer.assert_reasoning_matches_parse(&session) +} diff --git a/llama-cpp-bindings-tests/tests/reranking.rs b/llama-cpp-bindings-tests/tests/reranking.rs new file mode 100644 index 000000000..75253c997 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/reranking.rs @@ -0,0 +1,147 @@ +use std::time::Duration; + +use anyhow::Context; +use anyhow::Result; +use anyhow::bail; +use llama_cpp_bindings::SampledTokenClassifier; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::ggml_time_us; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings_tests::normalize::normalize; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +fn cosine_similarity(vec_a: &[f32], vec_b: &[f32]) -> f32 { + vec_a + .iter() + .zip(vec_b.iter()) + .map(|(left, right)| left * right) + .sum::() +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + n_seq_max = 2, + n_threads_batch = 8, + embeddings = true, +)] +fn reranking_produces_scores(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + + let query = "What is machine learning?"; + let documents = [ + "Machine learning is a subset of artificial intelligence.", + "The weather today is sunny and warm.", + ]; + + let document_count = documents.len(); + assert_eq!( + u32::try_from(document_count)?, + fixture.context_params.n_seq_max, + "attribute n_seq_max must match the document count this trial expects", + ); + + let mut ctx = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + ) + .with_context(|| "unable to create context")?; + + let prompt_lines: Vec = documents + .iter() + .map(|document| format!("{query}{document}")) + .collect(); + + let tokens_lines_list = prompt_lines + .iter() + .map(|line| model.str_to_token(line, AddBos::Always, ParseSpecialTokens::Always)) + .collect::, _>>() + .with_context(|| "failed to tokenize prompts")?; + + let n_ctx = usize::try_from(ctx.n_ctx())?; + + if tokens_lines_list.iter().any(|tokens| n_ctx < tokens.len()) { + bail!("one of the provided prompts exceeds the size of the context window"); + } + + let mut classifier = SampledTokenClassifier::undeterminable(model); + let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); + let t_main_start = ggml_time_us(); + + for (sequence_index, tokens) in tokens_lines_list.iter().enumerate() { + classifier.feed_prompt_sequence_to_batch( + &mut batch, + tokens, + i32::try_from(sequence_index)?, + false, + )?; + } + + let total_tokens: usize = tokens_lines_list.iter().map(Vec::len).sum(); + let total_token_count = u64::try_from(total_tokens)?; + + assert_eq!(classifier.pending_prompt_tokens(), total_token_count); + assert_eq!(classifier.usage().prompt_tokens, 0); + + ctx.clear_kv_cache(); + ctx.decode(&batch) + .with_context(|| "llama_decode() failed")?; + + let promoted = classifier.commit_prompt_tokens(); + assert_eq!(promoted, total_token_count); + + let mut embeddings = Vec::with_capacity(document_count); + + for sequence_index in 0..document_count { + let raw_embedding = ctx + .embeddings_seq_ith(i32::try_from(sequence_index)?) + .with_context(|| "failed to get sequence embeddings")?; + embeddings.push(normalize(raw_embedding)); + } + + let t_main_end = ggml_time_us(); + let duration = Duration::from_micros(u64::try_from(t_main_end - t_main_start)?); + + let tokens_per_second = f64::from(u32::try_from(total_tokens)?) / duration.as_secs_f64(); + + eprintln!( + "created embeddings for {total_tokens} tokens in {:.2} s, speed {tokens_per_second:.2} t/s", + duration.as_secs_f64(), + ); + + assert_eq!( + embeddings.len(), + document_count, + "should produce one embedding per document" + ); + + for (index, embedding) in embeddings.iter().enumerate() { + assert!( + !embedding.is_empty(), + "embedding {index} should not be empty" + ); + } + + let similarity = cosine_similarity(&embeddings[0], &embeddings[1]); + eprintln!("cosine similarity between document embeddings: {similarity:.4}"); + + assert!( + similarity.is_finite(), + "cosine similarity should be a finite number" + ); + + let usage = classifier.into_usage(); + assert_eq!(usage.prompt_tokens, total_token_count); + assert_eq!(usage.completion_tokens(), 0); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs b/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs new file mode 100644 index 000000000..daacfd9fa --- /dev/null +++ b/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs @@ -0,0 +1,205 @@ +use anyhow::Context as _; +use anyhow::Result; +use anyhow::bail; +use llama_cpp_bindings::BareJsonToolCalls; +use llama_cpp_bindings::GenerationProgress; +use llama_cpp_bindings::SampledToken; +use llama_cpp_bindings::SampledTokenClassifier; +use llama_cpp_bindings::TokenPiece; +use llama_cpp_bindings::ingest_outcome::IngestOutcome; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token_section::SampledTokenSection; +use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn chat_session_classifier_starts_in_the_pending_section(fixture: &LlamaFixture<'_>) -> Result<()> { + let session = user_turn_chat_session(fixture.model, "hello", "[]", false)?; + let classifier = SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + + assert_eq!(classifier.current_section(), SampledTokenSection::Pending); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn undeterminable_classifier_releases_each_token_with_its_visible_and_raw_piece( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let mut classifier = SampledTokenClassifier::undeterminable(model); + let [ordinary_token] = + model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?[..] + else { + bail!("\"hello\" must be a single token"); + }; + + let mut outcomes = Vec::new(); + classifier.ingest(ordinary_token, &mut outcomes)?; + classifier.finish(&mut outcomes)?; + + assert_eq!( + outcomes, + [IngestOutcome { + sampled_token: SampledToken::Undeterminable(ordinary_token), + piece: TokenPiece::Visible("hello".to_owned()), + }] + ); + assert_eq!(classifier.usage().undeterminable_tokens, 1); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn finishing_a_chat_answer_settles_its_pending_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let session = user_turn_chat_session(model, "hello", "[]", false)?; + let mut classifier = + SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let [ordinary_token] = + model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?[..] + else { + bail!("\"hello\" must be a single token"); + }; + + let mut outcomes = Vec::new(); + classifier.ingest(ordinary_token, &mut outcomes)?; + classifier.finish(&mut outcomes)?; + + assert_eq!( + outcomes, + [IngestOutcome { + sampled_token: SampledToken::Content(ordinary_token), + piece: TokenPiece::Visible("hello".to_owned()), + }] + ); + assert_eq!(classifier.usage().content_tokens, 1); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn ingest_counts_every_token_through_the_end_of_generation( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let mut classifier = SampledTokenClassifier::undeterminable(model); + let [ordinary_token] = + model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?[..] + else { + bail!("\"hello\" must be a single token"); + }; + let mut outcomes = Vec::new(); + + classifier.ingest(ordinary_token, &mut outcomes)?; + + assert_eq!( + classifier.ingest( + model.token_eos().context("model declares an EOS token")?, + &mut outcomes + )?, + GenerationProgress::Ended + ); + + assert_eq!(classifier.usage().undeterminable_tokens, 2); + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs b/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs new file mode 100644 index 000000000..e18f86923 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs @@ -0,0 +1,95 @@ +use anyhow::Context as _; +use anyhow::Result; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn accept_many_iterates_over_borrowed_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut sampler = LlamaSampler::chain_simple([LlamaSampler::greedy()?])?; + let tokens = vec![ + fixture + .model + .token_bos() + .context("model declares a BOS token")?, + fixture + .model + .token_eos() + .context("model declares an EOS token")?, + ]; + + sampler.accept_many(&tokens)?; + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn with_tokens_returns_self_after_accepting_each_token(fixture: &LlamaFixture<'_>) -> Result<()> { + let sampler = LlamaSampler::chain_simple([LlamaSampler::greedy()?])?; + let tokens = [ + fixture + .model + .token_bos() + .context("model declares a BOS token")?, + fixture + .model + .token_eos() + .context("model declares an EOS token")?, + ]; + + let _consumed = sampler.with_tokens(tokens.iter().copied())?; + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn accept_consumes_a_single_token(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut sampler = LlamaSampler::chain_simple([LlamaSampler::greedy()?])?; + + sampler.accept( + fixture + .model + .token_bos() + .context("model declares a BOS token")?, + )?; + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn accept_succeeds_for_a_valid_token(_fixture: &LlamaFixture<'_>) -> Result<()> { + let mut sampler = LlamaSampler::chain_simple([LlamaSampler::greedy()?])?; + + sampler.accept(LlamaToken::new(0))?; + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/session_file_path_validation.rs b/llama-cpp-bindings-tests/tests/session_file_path_validation.rs new file mode 100644 index 000000000..ccc5a5a34 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/session_file_path_validation.rs @@ -0,0 +1,363 @@ +use anyhow::Result; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { + use std::ffi::OsStr; + use std::os::unix::ffi::OsStrExt; + + let context = fixture.build_context()?; + + let non_utf8_path = std::path::Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); + let result = context.state_save_file(non_utf8_path, &[]); + + assert!(result.is_err()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { + use std::ffi::OsStr; + use std::os::unix::ffi::OsStrExt; + + let mut context = fixture.build_context()?; + + let non_utf8_path = std::path::Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); + let result = context.state_load_file(non_utf8_path, 512); + + assert!(result.is_err()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_seq_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { + use std::ffi::OsStr; + use std::os::unix::ffi::OsStrExt; + + let context = fixture.build_context()?; + + let non_utf8_path = std::path::Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); + let result = context.state_seq_save_file(non_utf8_path, 0, &[]); + + assert!(result.is_err()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_seq_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { + use std::ffi::OsStr; + use std::os::unix::ffi::OsStrExt; + + let mut context = fixture.build_context()?; + + let non_utf8_path = std::path::Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); + let result = context.state_seq_load_file(non_utf8_path, 0, 512); + + assert!(result.is_err()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_save_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = fixture.build_context()?; + + let path_with_null = std::path::Path::new("/tmp/foo\0bar.bin"); + let result = context.state_save_file(path_with_null, &[]); + + assert!(result.is_err()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_load_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let path_with_null = std::path::Path::new("/tmp/foo\0bar.bin"); + let result = context.state_load_file(path_with_null, 512); + + assert!(result.is_err()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_seq_save_file_with_null_byte_in_path_returns_error( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let context = fixture.build_context()?; + + let path_with_null = std::path::Path::new("/tmp/foo\0bar.bin"); + let result = context.state_seq_save_file(path_with_null, 0, &[]); + + assert!(result.is_err()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_seq_load_file_with_null_byte_in_path_returns_error( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + let path_with_null = std::path::Path::new("/tmp/foo\0bar.bin"); + let result = context.state_seq_load_file(path_with_null, 0, 512); + + assert!(result.is_err()); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs b/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs new file mode 100644 index 000000000..02e76bdb9 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs @@ -0,0 +1,231 @@ +use anyhow::Result; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_load_file_with_zero_max_tokens_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let tokens = + fixture + .model + .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let session_path = std::env::temp_dir().join("llama_test_session_zero_max.bin"); + context.state_save_file(&session_path, &tokens)?; + + let result = context.state_load_file(&session_path, 0); + + assert!(result.is_err()); + let _ = std::fs::remove_file(&session_path); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_seq_load_file_with_zero_max_tokens_returns_error( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + let tokens = + fixture + .model + .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let session_path = std::env::temp_dir().join("llama_test_seq_state_zero_max.bin"); + context.state_seq_save_file(&session_path, 0, &tokens)?; + + let result = context.state_seq_load_file(&session_path, 0, 0); + + assert!(result.is_err()); + let _ = std::fs::remove_file(&session_path); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_load_file_with_insufficient_max_tokens_returns_length_error( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + let tokens = fixture.model.str_to_token( + "Hello world this is a longer string for more tokens", + AddBos::Always, + ParseSpecialTokens::Always, + )?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let session_path = std::env::temp_dir().join("llama_test_session_insuf.bin"); + context.state_save_file(&session_path, &tokens)?; + + let result = context.state_load_file(&session_path, 1); + + assert!(result.is_err()); + let _ = std::fs::remove_file(&session_path); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_seq_load_file_with_insufficient_max_tokens_returns_length_error( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + let tokens = fixture.model.str_to_token( + "Hello world this is a longer string for more tokens", + AddBos::Always, + ParseSpecialTokens::Always, + )?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let session_path = std::env::temp_dir().join("llama_test_seq_state_insuf.bin"); + context.state_seq_save_file(&session_path, 0, &tokens)?; + + let result = context.state_seq_load_file(&session_path, 0, 1); + + assert!(result.is_err()); + let _ = std::fs::remove_file(&session_path); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/session_files.rs b/llama-cpp-bindings-tests/tests/session_files.rs new file mode 100644 index 000000000..5c286b91b --- /dev/null +++ b/llama-cpp-bindings-tests/tests/session_files.rs @@ -0,0 +1,295 @@ +use anyhow::Result; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_save_and_load_file_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let tokens = + fixture + .model + .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let session_path = std::env::temp_dir().join("llama_test_session.bin"); + context.state_save_file(&session_path, &tokens)?; + + let loaded_tokens = context.state_load_file(&session_path, 512)?; + assert_eq!(loaded_tokens, tokens); + + std::fs::remove_file(&session_path)?; + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_seq_save_and_load_file_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let tokens = + fixture + .model + .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let session_path = std::env::temp_dir().join("llama_test_seq_state.bin"); + let bytes_written = context.state_seq_save_file(&session_path, 0, &tokens)?; + assert!(bytes_written > 0); + + let loaded = context.state_seq_load_file(&session_path, 0, 512)?; + + assert_eq!(loaded.tokens, tokens); + assert_eq!( + loaded.bytes_read, bytes_written, + "loading must consume exactly the bytes the save produced" + ); + + std::fs::remove_file(&session_path)?; + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_load_file_with_nonexistent_file_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let result = context.state_load_file("/nonexistent/session.bin", 512); + + assert!(result.is_err()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_seq_load_file_with_nonexistent_file_returns_error( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + let result = context.state_seq_load_file("/nonexistent/seq_state.bin", 0, 512); + + assert!(result.is_err()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_save_file_to_invalid_directory_returns_failed_to_save( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let context = fixture.build_context()?; + + let result = context.state_save_file("/nonexistent_dir/session.bin", &[]); + + assert!(result.is_err()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_seq_save_file_to_invalid_directory_returns_failed_to_save( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let context = fixture.build_context()?; + + let result = context.state_seq_save_file("/nonexistent_dir/seq_state.bin", 0, &[]); + + assert!(result.is_err()); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/special_tokens.rs b/llama-cpp-bindings-tests/tests/special_tokens.rs new file mode 100644 index 000000000..dc63ee1d7 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/special_tokens.rs @@ -0,0 +1,197 @@ +use anyhow::Context; +use anyhow::Result; +use llama_cpp_bindings::SampledToken; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn eos_token_differs_from_bos_and_ends_generation(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let eos = model.token_eos().context("model declares an EOS token")?; + + assert_ne!(model.token_bos(), Some(eos)); + assert!(model.is_eog_token(&SampledToken::Content(eos))); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 64, + embeddings = true, +)] +fn model_without_a_bos_token_reports_none(fixture: &LlamaFixture<'_>) { + assert_eq!(fixture.model.token_bos(), None); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_nl_is_the_first_token_of_a_tokenized_newline(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let newline_tokens = model.str_to_token("\n", AddBos::Never, ParseSpecialTokens::Never)?; + + assert_eq!(model.token_nl(), newline_tokens.first().copied()); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn decoder_only_model_has_no_decoder_start_token(fixture: &LlamaFixture<'_>) { + assert_eq!(fixture.model.decode_start_token(), None); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn chat_model_has_no_separator_token(fixture: &LlamaFixture<'_>) { + assert_eq!(fixture.model.token_sep(), None); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn deepseek_r1_8b_adds_a_bos_token(fixture: &LlamaFixture<'_>) { + assert!(fixture.model.adds_bos_token()); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_does_not_add_a_bos_token(fixture: &LlamaFixture<'_>) { + assert!(!fixture.model.adds_bos_token()); +} diff --git a/llama-cpp-bindings-tests/tests/state_snapshots.rs b/llama-cpp-bindings-tests/tests/state_snapshots.rs new file mode 100644 index 000000000..9c04e2988 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/state_snapshots.rs @@ -0,0 +1,257 @@ +use anyhow::Result; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn get_state_size_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = fixture.build_context()?; + + assert!(context.get_state_size() > 0); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn set_state_data_rejects_a_truncated_snapshot(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let tokens = + fixture + .model + .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let mut state_data = vec![0u8; context.get_state_size()]; + let bytes_copied = unsafe { context.copy_state_data(&mut state_data) }?; + state_data.truncate(bytes_copied / 2); + + assert_eq!( + unsafe { context.set_state_data(&state_data) }, + Err( + llama_cpp_bindings::context::state_data_error::StateDataError::NothingRestored { + provided_bytes: state_data.len(), + } + ), + "a truncated snapshot restores nothing, and that must not be reported as success" + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn copy_state_data_and_set_state_data_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let tokens = + fixture + .model + .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let state_size = context.get_state_size(); + let mut state_data = vec![0u8; state_size]; + let bytes_copied = unsafe { context.copy_state_data(&mut state_data) }?; + assert!(bytes_copied > 0); + + let bytes_read = unsafe { context.set_state_data(&state_data) }?; + assert_eq!( + bytes_read, bytes_copied, + "restoring the state must consume exactly the bytes the snapshot produced" + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_seq_get_size_ext_returns_size_for_decoded_sequence( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + use llama_cpp_bindings::context::llama_state_seq_flags::LlamaStateSeqFlags; + + let mut context = fixture.build_context()?; + + let tokens = + fixture + .model + .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let flags = LlamaStateSeqFlags::empty(); + let size = context.state_seq_get_size_ext(0, &flags); + + assert!(size > 0); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_seq_get_data_ext_and_set_data_ext_round_trip(fixture: &LlamaFixture<'_>) -> Result<()> { + use llama_cpp_bindings::context::llama_state_seq_flags::LlamaStateSeqFlags; + + let mut context = fixture.build_context()?; + + let tokens = + fixture + .model + .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let flags = LlamaStateSeqFlags::empty(); + let size = context.state_seq_get_size_ext(0, &flags); + let mut buffer = vec![0u8; size]; + let bytes_written = unsafe { context.state_seq_get_data_ext(&mut buffer, 0, &flags) }?; + + assert!(bytes_written > 0); + + let bytes_read = unsafe { context.state_seq_set_data_ext(&buffer, 0, &flags) }?; + + assert_eq!( + bytes_read, bytes_written, + "restoring the sequence state must consume exactly the bytes the snapshot produced" + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/structured_chat_output.rs b/llama-cpp-bindings-tests/tests/structured_chat_output.rs deleted file mode 100644 index 1de7a11cc..000000000 --- a/llama-cpp-bindings-tests/tests/structured_chat_output.rs +++ /dev/null @@ -1,2357 +0,0 @@ -use anyhow::Context; -use anyhow::Result; -use anyhow::bail; -use llama_cpp_bindings::BareJsonToolCalls; -use llama_cpp_bindings::ChatMessageParseOutcome; -use llama_cpp_bindings::ChatMessageParser; -use llama_cpp_bindings::GenerationProgress; -use llama_cpp_bindings::MarkerRole; -use llama_cpp_bindings::ParsedChatMessage; -use llama_cpp_bindings::ParsedToolCall; -use llama_cpp_bindings::ParsedToolCallId; -use llama_cpp_bindings::SampledTokenSection; -use llama_cpp_bindings::TokenUsage; -use llama_cpp_bindings::ToolCallArgsShape; -use llama_cpp_bindings::ToolCallArguments; -use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_bindings::context::context_limits::ContextLimits; -use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::LlamaChatMessage; -use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::model::ParseSpecialTokens; -use llama_cpp_bindings::sampling::LlamaSampler; -use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; -use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoopOutcome; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; -use serde_json::Value; -use serde_json::json; - -fn parse_partial_reasoning_response( - model: &LlamaModel, - generated: &str, -) -> Result { - let Some(markers) = model.reasoning_markers()? else { - bail!("model must expose reasoning markers"); - }; - let response = if generated.trim_start().starts_with(markers.open.trim()) { - generated.to_owned() - } else { - format!("{}{generated}", markers.open) - }; - let parse_outcome = ChatMessageParser::new(model, "[]")?.parse(&response, true)?; - let ChatMessageParseOutcome::Recognized(parsed) = parse_outcome else { - bail!("model chat template must recognize a partial reasoning response"); - }; - Ok(parsed) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn deepseek_r1_8b_classifier_does_not_emit_reasoning_for_thinking_disabled_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 200; - - const DEEPSEEK_R1_8B_THINKING_DISABLED_PROMPT: &str = "\ -<｜User｜>What is 2 + 2?<｜Assistant｜> - - - -"; - - const FORBIDDEN_MARKERS: &[&str] = &["", ""]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - DEEPSEEK_R1_8B_THINKING_DISABLED_PROMPT, - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - context.decode(&batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - - assert!( - !outcome.generated_raw.is_empty(), - "DeepSeek-R1-8B: must generate at least one token" - ); - assert_eq!( - outcome.observed_reasoning, 0, - "DeepSeek-R1-8B thinking-disabled: classifier must not emit any Reasoning token \ - when the prompt closes the think block before generation begins; \ - generated={:?}", - outcome.generated_raw - ); - assert_eq!( - outcome.observed_undeterminable, 0, - "DeepSeek-R1-8B thinking-disabled: prompt-token replay must move section to Content \ - before generation, so no Undeterminable tokens may be emitted; \ - generated={:?}", - outcome.generated_raw - ); - assert_eq!( - usage.reasoning_tokens, 0, - "DeepSeek-R1-8B thinking-disabled: usage.reasoning_tokens must be zero; usage={usage:?}" - ); - assert_eq!( - usage.undeterminable_tokens, 0, - "DeepSeek-R1-8B thinking-disabled: usage.undeterminable_tokens must be zero; usage={usage:?}" - ); - assert!( - outcome.observed_content > 0, - "DeepSeek-R1-8B thinking-disabled: classifier must emit at least one Content token" - ); - assert_eq!( - usage.completion_tokens(), - outcome.observed_content, - "DeepSeek-R1-8B thinking-disabled: completion tokens must equal observed Content tokens" - ); - - for forbidden in FORBIDDEN_MARKERS { - assert!( - !outcome.content_stream.contains(forbidden), - "DeepSeek-R1-8B thinking-disabled: content_stream leaked marker {forbidden:?}; \ - content_stream={:?}", - outcome.content_stream - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn deepseek_r1_8b_classifier_emits_reasoning_for_thinking_enabled_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 512; - - const DEEPSEEK_R1_8B_THINKING_PROMPT: &str = "\ -<｜User｜>What is 2 + 2?<｜Assistant｜> -"; - - const FORBIDDEN_MARKERS: &[&str] = &["", ""]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - DEEPSEEK_R1_8B_THINKING_PROMPT, - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - context.decode(&batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - let parsed = parse_partial_reasoning_response(model, &outcome.generated_raw)?; - - assert_deepseek_r1_token_counts(&outcome, usage); - assert_deepseek_r1_streams(&outcome, &parsed, MAX_GENERATED_TOKENS, FORBIDDEN_MARKERS); - - Ok(()) -} - -fn assert_deepseek_r1_token_counts(outcome: &ClassifySampleLoopOutcome, usage: &TokenUsage) { - assert!( - !outcome.generated_raw.is_empty(), - "DeepSeek-R1-8B: must generate at least one token" - ); - assert!( - outcome.observed_reasoning > 0, - "DeepSeek-R1-8B: classifier must emit at least one Reasoning token when the prompt \ - opens a block; outcome={outcome:?}", - ); - assert!( - usage.reasoning_tokens > 0, - "DeepSeek-R1-8B: usage.reasoning_tokens must be non-zero when the prompt opens a \ - block; usage was {usage:?}" - ); - assert_eq!( - outcome.observed_undeterminable, 0, - "DeepSeek-R1-8B: prompt-token replay must move section to Reasoning before generation, \ - so no Undeterminable tokens may be emitted; outcome={outcome:?}" - ); - assert_eq!( - usage.undeterminable_tokens, 0, - "DeepSeek-R1-8B: usage.undeterminable_tokens must be zero; usage={usage:?}" - ); - assert_eq!( - usage.completion_tokens(), - outcome.observed_content + outcome.observed_reasoning, - "DeepSeek-R1-8B: completion tokens must equal observed Content + Reasoning" - ); -} - -fn assert_deepseek_r1_streams( - outcome: &ClassifySampleLoopOutcome, - parsed: &ParsedChatMessage, - max_generated_tokens: i32, - forbidden_markers: &[&str], -) { - assert!( - !parsed.reasoning_content.is_empty(), - "DeepSeek-R1-8B partial response must expose reasoning within {max_generated_tokens} tokens" - ); - assert_eq!( - outcome.reasoning_stream, parsed.reasoning_content, - "DeepSeek-R1-8B: per-token reasoning stream must equal parser-side reasoning_content \ - (any difference means a marker leaked into the user-visible stream)", - ); - assert_eq!( - outcome.content_stream, parsed.content, - "DeepSeek-R1-8B: per-token content stream must equal parser-side content \ - (any difference means a marker leaked into the user-visible stream)", - ); - - for forbidden in forbidden_markers { - assert!( - !outcome.reasoning_stream.contains(forbidden), - "DeepSeek-R1-8B: reasoning_stream leaked marker {forbidden:?}; \ - reasoning_stream={:?}", - outcome.reasoning_stream - ); - assert!( - !outcome.content_stream.contains(forbidden), - "DeepSeek-R1-8B: content_stream leaked marker {forbidden:?}; \ - content_stream={:?}", - outcome.content_stream - ); - } -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn deepseek_r1_8b_duck_types_gemma_paired_quote(fixture: &LlamaFixture<'_>) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const GEMMA_PAIRED_QUOTE_PAYLOAD: &str = - "<|tool_call>call:get_weather{location:<|\"|>Paris<|\"|>}"; - - let outcome = ChatMessageParser::new(fixture.model, TOOLS_JSON)? - .parse(GEMMA_PAIRED_QUOTE_PAYLOAD, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!( - "duck-type pass must recognise Gemma paired-quote on a model with no registered \ - template; got Unrecognized" - ); - }; - assert_eq!( - parsed.tool_calls.len(), - 1, - "expected one tool call; got {:?}", - parsed.tool_calls - ); - assert_eq!(parsed.tool_calls[0].name, "get_weather"); - let location = match &parsed.tool_calls[0].arguments { - ToolCallArguments::ValidJson(value) => value - .get("location") - .and_then(|value| value.as_str()) - .map(str::to_owned), - ToolCallArguments::InvalidJson(raw) => { - bail!("expected ValidJson, got InvalidJson: {raw}"); - } - }; - assert_eq!(location.as_deref(), Some("Paris")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn deepseek_r1_8b_duck_types_glm_key_value_tags(fixture: &LlamaFixture<'_>) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const GLM_KEY_VALUE_PAYLOAD: &str = "get_weather\ -location\ -Paris\ -"; - - let outcome = - ChatMessageParser::new(fixture.model, TOOLS_JSON)?.parse(GLM_KEY_VALUE_PAYLOAD, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!( - "duck-type pass must recognise GLM key-value tags on a model with no registered \ - template; got Unrecognized" - ); - }; - assert_eq!( - parsed.tool_calls.len(), - 1, - "expected one tool call; got {:?}", - parsed.tool_calls - ); - assert_eq!(parsed.tool_calls[0].name, "get_weather"); - let location = match &parsed.tool_calls[0].arguments { - ToolCallArguments::ValidJson(value) => value - .get("location") - .and_then(|value| value.as_str()) - .map(str::to_owned), - ToolCallArguments::InvalidJson(raw) => { - bail!("expected ValidJson, got InvalidJson: {raw}"); - } - }; - assert_eq!(location.as_deref(), Some("Paris")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn deepseek_r1_8b_duck_types_mistral_bracketed_json(fixture: &LlamaFixture<'_>) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const MISTRAL_BRACKETED_JSON_PAYLOAD: &str = - r#"[TOOL_CALLS]get_weather[ARGS]{"location":"Paris"}"#; - - let outcome = ChatMessageParser::new(fixture.model, TOOLS_JSON)? - .parse(MISTRAL_BRACKETED_JSON_PAYLOAD, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!( - "duck-type pass must recognise Mistral bracketed-JSON on a model with no registered \ - template; got Unrecognized" - ); - }; - assert_eq!( - parsed.tool_calls.len(), - 1, - "expected one tool call; got {:?}", - parsed.tool_calls - ); - assert_eq!(parsed.tool_calls[0].name, "get_weather"); - let location = match &parsed.tool_calls[0].arguments { - ToolCallArguments::ValidJson(value) => value - .get("location") - .and_then(|value| value.as_str()) - .map(str::to_owned), - ToolCallArguments::InvalidJson(raw) => { - bail!("expected ValidJson, got InvalidJson: {raw}"); - } - }; - assert_eq!(location.as_deref(), Some("Paris")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn deepseek_r1_8b_duck_types_qwen_xml(fixture: &LlamaFixture<'_>) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const QWEN_XML_PAYLOAD: &str = "\n\ -\n\ -\n\ -Paris\n\ -\n\ -\n\ -"; - - let outcome = - ChatMessageParser::new(fixture.model, TOOLS_JSON)?.parse(QWEN_XML_PAYLOAD, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!( - "duck-type pass must recognise Qwen XML on a model with no registered template; \ - got Unrecognized" - ); - }; - assert_eq!( - parsed.tool_calls.len(), - 1, - "expected one tool call; got {:?}", - parsed.tool_calls - ); - assert_eq!(parsed.tool_calls[0].name, "get_weather"); - let location = match &parsed.tool_calls[0].arguments { - ToolCallArguments::ValidJson(value) => value - .get("location") - .and_then(|value| value.as_str()) - .map(str::to_owned), - ToolCallArguments::InvalidJson(raw) => { - bail!("expected ValidJson, got InvalidJson: {raw}"); - } - }; - assert_eq!(location.as_deref(), Some("Paris")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn deepseek_r1_8b_recognizes_empty_tool_calls_when_input_is_plain_content_with_tools_requested( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const PLAIN_CONTENT: &str = "Sorry, I cannot help with that."; - - let outcome = ChatMessageParser::new(fixture.model, TOOLS_JSON)?.parse(PLAIN_CONTENT, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!( - "plain content with tools requested must produce Recognized (with empty tool_calls); \ - got Unrecognized" - ); - }; - assert!( - parsed.tool_calls.is_empty(), - "expected no tool calls; got {:?}", - parsed.tool_calls - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn deepseek_r1_8b_recognizes_empty_tool_calls_when_tools_not_requested( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const PLAIN_CONTENT: &str = "Hello there."; - - let outcome = ChatMessageParser::new(fixture.model, "[]")?.parse(PLAIN_CONTENT, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!("plain content with empty tools array must produce Recognized; got Unrecognized"); - }; - assert!( - parsed.tool_calls.is_empty(), - "expected no tool calls; got {:?}", - parsed.tool_calls - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn gemma4_classifier_does_not_emit_reasoning_for_thinking_disabled_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 200; - - const GEMMA4_THINKING_DISABLED_PROMPT: &str = "\ -user\nReply with the single word: four. Do not explain.\n\ -model\n<|channel>thought\n\n"; - - const FORBIDDEN_MARKERS: &[&str] = &["<|channel>thought", ""]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - GEMMA4_THINKING_DISABLED_PROMPT, - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - context.decode(&batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::greedy()?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - - assert!( - !outcome.generated_raw.is_empty(), - "Gemma 4 must generate at least one token" - ); - assert_eq!( - outcome.observed_reasoning, 0, - "Gemma 4 thinking-disabled: classifier must not emit any Reasoning token \ - when the prompt closes the thought channel before generation begins; \ - generated={:?}", - outcome.generated_raw - ); - assert_eq!( - outcome.observed_undeterminable, 0, - "Gemma 4 thinking-disabled: prompt-token replay must move section to Content \ - before generation, so no Undeterminable tokens may be emitted; \ - generated={:?}", - outcome.generated_raw - ); - assert_eq!( - usage.reasoning_tokens, 0, - "Gemma 4 thinking-disabled: usage.reasoning_tokens must be zero; usage={usage:?}" - ); - assert_eq!( - usage.undeterminable_tokens, 0, - "Gemma 4 thinking-disabled: usage.undeterminable_tokens must be zero; usage={usage:?}" - ); - assert!( - outcome.observed_content > 0, - "Gemma 4 thinking-disabled: classifier must emit at least one Content token" - ); - assert_eq!( - usage.completion_tokens(), - outcome.observed_content, - "Gemma 4 thinking-disabled: completion tokens must equal observed Content tokens" - ); - - for forbidden in FORBIDDEN_MARKERS { - assert!( - !outcome.content_stream.contains(forbidden), - "Gemma 4 thinking-disabled: content_stream leaked marker {forbidden:?}; \ - content_stream={:?}", - outcome.content_stream - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn gemma4_classifier_emits_reasoning_for_thinking_prompt(fixture: &LlamaFixture<'_>) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 512; - - const GEMMA4_THINKING_PROMPT: &str = "\ -user\nReply with the single word: four. Do not explain.\n\ -model\n<|channel>thought\n"; - - const FORBIDDEN_MARKERS: &[&str] = &["<|channel>thought", ""]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - GEMMA4_THINKING_PROMPT, - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - context.decode(&batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::greedy()?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - let parsed = parse_partial_reasoning_response(model, &outcome.generated_raw)?; - - assert!( - !outcome.generated_raw.is_empty(), - "Gemma 4 must generate at least one token" - ); - assert!( - outcome.observed_reasoning > 0, - "Gemma 4 classifier must emit at least one Reasoning token when the model \ - emits a `<|channel>thought` block; outcome={outcome:?}", - ); - assert!( - usage.reasoning_tokens > 0, - "Gemma 4 usage.reasoning_tokens must be non-zero when the model emits a \ - reasoning block; usage was {usage:?}" - ); - assert_eq!( - outcome.observed_undeterminable, 0, - "Gemma 4: classifier must not emit Undeterminable when the model emits a \ - detected `<|channel>thought` marker; outcome={outcome:?}" - ); - assert_eq!( - usage.undeterminable_tokens, 0, - "Gemma 4: usage.undeterminable_tokens must be zero; usage={usage:?}" - ); - assert_eq!( - usage.completion_tokens(), - outcome.observed_content + outcome.observed_reasoning, - "Gemma 4: completion tokens must equal observed Content + Reasoning" - ); - assert!( - !parsed.reasoning_content.is_empty(), - "Gemma 4 partial response must expose reasoning within {MAX_GENERATED_TOKENS} tokens" - ); - assert_eq!(outcome.reasoning_stream, parsed.reasoning_content); - assert_eq!(outcome.content_stream, parsed.content); - - for forbidden in FORBIDDEN_MARKERS { - assert!( - !outcome.reasoning_stream.contains(forbidden), - "Gemma 4: reasoning_stream leaked marker {forbidden:?}; \ - reasoning_stream={:?}", - outcome.reasoning_stream - ); - assert!( - !outcome.content_stream.contains(forbidden), - "Gemma 4: content_stream leaked marker {forbidden:?}; \ - content_stream={:?}", - outcome.content_stream - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn gemma4_parses_tool_call_payload(fixture: &LlamaFixture<'_>) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const GEMMA4_PAIRED_QUOTE_PAYLOAD: &str = - "<|tool_call>call:get_weather{location:<|\"|>Paris<|\"|>}"; - - let outcome = ChatMessageParser::new(fixture.model, TOOLS_JSON)? - .parse(GEMMA4_PAIRED_QUOTE_PAYLOAD, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!("expected Recognized for Gemma 4 PairedQuote on a Gemma-4 model; got Unrecognized"); - }; - assert_eq!( - parsed.tool_calls.len(), - 1, - "expected one tool call; got {:?}", - parsed.tool_calls - ); - assert_eq!(parsed.tool_calls[0].name, "get_weather"); - let location = match &parsed.tool_calls[0].arguments { - ToolCallArguments::ValidJson(value) => value - .get("location") - .and_then(|value| value.as_str()) - .map(str::to_owned), - ToolCallArguments::InvalidJson(raw) => { - bail!("expected ValidJson, got InvalidJson: {raw}"); - } - }; - assert_eq!(location.as_deref(), Some("Paris")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn gemma4_template_format_returns_full_markers(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let template = model - .chat_template(None) - .expect("Gemma 4 chat template must be present"); - let template_str = template.to_str().expect("template must be valid UTF-8"); - assert!( - template_str.contains("<|tool_call>call:"), - "Gemma 4 chat template must contain '<|tool_call>call:' fingerprint; \ - template starts with: {:?}", - &template_str[..template_str.len().min(200)], - ); - - let markers = model - .tool_call_markers()? - .expect("Gemma 4 must produce ToolCallMarkers from its template format"); - - assert_eq!(markers.open, "<|tool_call>call:"); - assert_eq!(markers.close, "}"); - let ToolCallArgsShape::PairedQuote(shape) = markers.args_shape else { - panic!("expected PairedQuote variant, got {:?}", markers.args_shape); - }; - assert_eq!(shape.name_args_separator, "{"); - assert_eq!(shape.value_quote.open, "<|\"|>"); - assert_eq!(shape.value_quote.close, "<|\"|>"); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn glm47_classifier_does_not_emit_reasoning_for_thinking_disabled_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 200; - - const GLM47_THINKING_DISABLED_PROMPT: &str = "\ -<|user|> -What is 2 + 2? -<|assistant|> - - -"; - - const FORBIDDEN_MARKERS: &[&str] = &["", ""]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - GLM47_THINKING_DISABLED_PROMPT, - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - context.decode(&batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - - assert_ne!(outcome.generated_raw, ""); - assert_eq!(outcome.observed_reasoning, 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(usage.reasoning_tokens, 0); - assert_eq!(usage.undeterminable_tokens, 0); - assert!(outcome.observed_content > 0); - assert_eq!(usage.completion_tokens(), outcome.observed_content); - - for forbidden in FORBIDDEN_MARKERS { - assert!(!outcome.content_stream.contains(forbidden)); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn glm47_classifier_emits_reasoning_for_thinking_enabled_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 512; - - const GLM47_THINKING_PROMPT: &str = "\ -<|user|> -What is 2 + 2? -<|assistant|> - -"; - - const FORBIDDEN_MARKERS: &[&str] = &["", ""]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - GLM47_THINKING_PROMPT, - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - context.decode(&batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - let parsed = parse_partial_reasoning_response(model, &outcome.generated_raw)?; - - assert_ne!(outcome.generated_raw, ""); - assert!(outcome.observed_reasoning > 0); - assert!(usage.reasoning_tokens > 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(usage.undeterminable_tokens, 0); - assert_eq!( - usage.completion_tokens(), - outcome.observed_content + outcome.observed_reasoning - ); - - assert!( - !parsed.reasoning_content.is_empty(), - "GLM-4.7 partial response must expose reasoning within {MAX_GENERATED_TOKENS} tokens" - ); - assert_eq!(outcome.reasoning_stream, parsed.reasoning_content); - assert_eq!(outcome.content_stream, parsed.content); - - for forbidden in FORBIDDEN_MARKERS { - assert!(!outcome.reasoning_stream.contains(forbidden)); - assert!(!outcome.content_stream.contains(forbidden)); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn glm47_parses_tool_call_payload(fixture: &LlamaFixture<'_>) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const GLM47_KEY_VALUE_PAYLOAD: &str = "get_weather\ -location\ -Paris\ -"; - - let outcome = - ChatMessageParser::new(fixture.model, TOOLS_JSON)?.parse(GLM47_KEY_VALUE_PAYLOAD, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!( - "expected Recognized for GLM-4.7 key-value tags on a GLM-4.7-Flash model; got Unrecognized" - ); - }; - assert_eq!(parsed.tool_calls.len(), 1); - assert_eq!(parsed.tool_calls[0].name, "get_weather"); - let location = match &parsed.tool_calls[0].arguments { - ToolCallArguments::ValidJson(value) => value - .get("location") - .and_then(|value| value.as_str()) - .map(str::to_owned), - ToolCallArguments::InvalidJson(raw) => { - bail!("expected ValidJson, got InvalidJson: {raw}"); - } - }; - assert_eq!(location.as_deref(), Some("Paris")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn glm47_template_format_returns_full_markers(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let template = model - .chat_template(None) - .expect("GLM-4.7 chat template must be present"); - let template_str = template.to_str().expect("template must be valid UTF-8"); - assert!(template_str.contains("")); - - let markers = model - .tool_call_markers()? - .expect("GLM-4.7 must produce ToolCallMarkers from its template format"); - - assert_eq!(markers.open, ""); - assert_eq!(markers.close, ""); - let ToolCallArgsShape::KeyValueXmlTags(shape) = markers.args_shape else { - panic!( - "expected KeyValueXmlTags variant, got {:?}", - markers.args_shape - ); - }; - assert_eq!(shape.key_open, ""); - assert_eq!(shape.key_close, ""); - assert_eq!(shape.value_open, ""); - assert_eq!(shape.value_close, ""); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn mistral3_classifier_does_not_emit_reasoning_for_thinking_disabled_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 200; - - const MISTRAL3_THINKING_DISABLED_PROMPT: &str = "\ -[INST]Reply with the single word: four. Do not explain.[/INST][THINK][/THINK]"; - - const FORBIDDEN_MARKERS: &[&str] = &["[THINK]", "[/THINK]"]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - MISTRAL3_THINKING_DISABLED_PROMPT, - AddBos::Always, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - context.decode(&batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::greedy()?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - - assert_ne!(outcome.generated_raw, ""); - assert_eq!(outcome.observed_reasoning, 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(usage.reasoning_tokens, 0); - assert_eq!(usage.undeterminable_tokens, 0); - assert!(outcome.observed_content > 0); - assert_eq!(usage.completion_tokens(), outcome.observed_content); - - for forbidden in FORBIDDEN_MARKERS { - assert!(!outcome.content_stream.contains(forbidden)); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn mistral3_classifier_emits_reasoning_for_thinking_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 512; - - const MISTRAL3_THINKING_PROMPT: &str = "\ -[SYSTEM_PROMPT]# HOW YOU SHOULD THINK AND ANSWER\n\n\ -First draft your thinking process (inner monologue) until you arrive at a response. \ -Format your response using Markdown, and use LaTeX for any mathematical equations. \ -Write both your thoughts and the response in the same language as the input.\n\n\ -Your thinking process must follow the template below:\ -[THINK]Your thoughts or/and draft, like working through an exercise on scratch paper. \ -Be as casual and as long as you want until you are confident to generate the response \ -to the user.[/THINK]Here, provide a self-contained response.[/SYSTEM_PROMPT]\ -[INST]Reply with the single word: four. Do not explain.[/INST]"; - - const FORBIDDEN_MARKERS: &[&str] = &["[THINK]", "[/THINK]"]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - MISTRAL3_THINKING_PROMPT, - AddBos::Always, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - context.decode(&batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::greedy()?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - let parsed = parse_partial_reasoning_response(model, &outcome.generated_raw)?; - - assert_ne!(outcome.generated_raw, ""); - assert!(outcome.observed_reasoning > 0); - assert!(usage.reasoning_tokens > 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(usage.undeterminable_tokens, 0); - assert_eq!( - usage.completion_tokens(), - outcome.observed_content + outcome.observed_reasoning, - ); - assert_ne!(parsed.reasoning_content, ""); - assert_eq!(outcome.reasoning_stream, parsed.reasoning_content); - assert_eq!(outcome.content_stream, parsed.content); - - for forbidden in FORBIDDEN_MARKERS { - assert!(!outcome.reasoning_stream.contains(forbidden)); - assert!(!outcome.content_stream.contains(forbidden)); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn mistral3_parses_tool_call_payload(fixture: &LlamaFixture<'_>) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const MISTRAL3_BRACKETED_JSON_PAYLOAD: &str = - r#"[TOOL_CALLS]get_weather[ARGS]{"location":"Paris"}"#; - - let outcome = ChatMessageParser::new(fixture.model, TOOLS_JSON)? - .parse(MISTRAL3_BRACKETED_JSON_PAYLOAD, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!( - "expected Recognized for Mistral 3 BracketedJson on a Mistral-3 model; got Unrecognized" - ); - }; - assert_eq!(parsed.tool_calls.len(), 1); - assert_eq!(parsed.tool_calls[0].name, "get_weather"); - let location = match &parsed.tool_calls[0].arguments { - ToolCallArguments::ValidJson(value) => value - .get("location") - .and_then(|value| value.as_str()) - .map(str::to_owned), - ToolCallArguments::InvalidJson(raw) => { - bail!("expected ValidJson, got InvalidJson: {raw}"); - } - }; - assert_eq!(location.as_deref(), Some("Paris")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128, -)] -fn qwen35_chat_inference_emits_reasoning_when_template_auto_opens( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let backend = fixture.backend; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let chat_template = model.chat_template(None)?; - let messages = vec![LlamaChatMessage::new( - "user".to_owned(), - "Hello! How are you?".to_owned(), - )?]; - let prompt = model.apply_chat_template(&chat_template, &messages, true, true)?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let tokens = model.str_to_token(&prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let prompt_token_count = u64::try_from(tokens.len())?; - - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; - - context.decode(&batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::greedy()?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: 512, - } - .run()?; - - assert_ne!(outcome.generated_raw, ""); - assert!(outcome.observed_reasoning > 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(outcome.observed_tool_call, 0); - - let usage = classifier.into_usage(); - assert_eq!(usage.prompt_tokens, prompt_token_count); - assert_eq!(usage.reasoning_tokens, outcome.observed_reasoning); - assert_eq!(usage.undeterminable_tokens, 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128, -)] -fn qwen35_shared_reasoning_close_and_tool_call_open_is_one_transition( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let reasoning_markers = fixture - .model - .reasoning_markers()? - .expect("Qwen3.5 must expose reasoning markers"); - let streaming_markers = fixture.model.streaming_markers()?; - - assert!(streaming_markers.iter().any(|marker| { - marker.roles().contains(&MarkerRole::ReasoningOpen) && !marker.tokens().is_empty() - })); - assert_eq!( - streaming_markers - .iter() - .filter(|marker| marker.roles().contains(&MarkerRole::ReasoningClose)) - .count(), - reasoning_markers.closes.len() - ); - - let reasoning_open = streaming_markers - .iter() - .find(|marker| marker.roles().contains(&MarkerRole::ReasoningOpen)) - .expect("Qwen3.5 must expose a reasoning opener") - .tokens() - .to_vec(); - let shared_boundary = streaming_markers - .iter() - .find(|marker| { - marker.roles().contains(&MarkerRole::ReasoningClose) - && marker.roles().contains(&MarkerRole::ToolCallOpen) - }) - .expect("Qwen3.5 must share its tool-call opener with a reasoning close") - .tokens() - .to_vec(); - - let mut classifier = fixture - .model - .sampled_token_classifier(BareJsonToolCalls::Detect)?; - classifier.ingest_prompt_tokens(&reasoning_open); - assert_eq!(classifier.current_section(), SampledTokenSection::Reasoning); - - classifier.ingest_prompt_tokens(&shared_boundary); - assert_eq!(classifier.current_section(), SampledTokenSection::ToolCall); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Ministral-3-3B-Reasoning-2512-GGUF", "Ministral-3-3B-Reasoning-2512-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128, -)] -fn mistral3_tool_call_turn_finished_in_the_prompt_leaves_the_classifier_pending( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let mut finished_tool_call_turn = model.str_to_token( - "[TOOL_CALLS]read[ARGS]{\"path\":\"marker.txt\"}", - AddBos::Never, - ParseSpecialTokens::Always, - )?; - - finished_tool_call_turn.push(model.token_eos().context("model declares an EOS token")?); - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - - classifier.ingest_prompt_tokens(&finished_tool_call_turn); - - assert_eq!(classifier.current_section(), SampledTokenSection::Pending); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn qwen35_classifier_does_not_emit_reasoning_for_thinking_disabled_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 200; - - const QWEN35_THINKING_DISABLED_PROMPT: &str = "\ -<|im_start|>user -What is 2 + 2?<|im_end|> -<|im_start|>assistant - - - - -"; - - const FORBIDDEN_MARKERS: &[&str] = &["", ""]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - QWEN35_THINKING_DISABLED_PROMPT, - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - context.decode(&batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - - assert_ne!(outcome.generated_raw, ""); - assert_eq!(outcome.observed_reasoning, 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(usage.reasoning_tokens, 0); - assert_eq!(usage.undeterminable_tokens, 0); - assert!(outcome.observed_content > 0); - assert_eq!(usage.completion_tokens(), outcome.observed_content); - - for forbidden in FORBIDDEN_MARKERS { - assert!(!outcome.content_stream.contains(forbidden)); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn qwen35_classifier_emits_reasoning_for_thinking_enabled_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 512; - - const QWEN35_THINKING_PROMPT: &str = "\ -<|im_start|>user -What is 2 + 2?<|im_end|> -<|im_start|>assistant - -"; - - const FORBIDDEN_MARKERS: &[&str] = &["", ""]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - QWEN35_THINKING_PROMPT, - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - context.decode(&batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - let parsed = parse_partial_reasoning_response(model, &outcome.generated_raw)?; - - assert_ne!(outcome.generated_raw, ""); - assert!(outcome.observed_reasoning > 0); - assert!(usage.reasoning_tokens > 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(usage.undeterminable_tokens, 0); - assert_eq!( - usage.completion_tokens(), - outcome.observed_content + outcome.observed_reasoning, - ); - - assert!( - !parsed.reasoning_content.is_empty(), - "Qwen3.5 must close its reasoning block within {MAX_GENERATED_TOKENS} tokens" - ); - assert_eq!(outcome.reasoning_stream, parsed.reasoning_content); - assert_eq!(outcome.content_stream, parsed.content); - - for forbidden in FORBIDDEN_MARKERS { - assert!(!outcome.reasoning_stream.contains(forbidden)); - assert!(!outcome.content_stream.contains(forbidden)); - } - - Ok(()) -} - -fn arguments_as_json(arguments: &ToolCallArguments) -> Result<&Value> { - match arguments { - ToolCallArguments::ValidJson(value) => Ok(value), - ToolCallArguments::InvalidJson(raw) => { - bail!("expected ValidJson arguments, got InvalidJson: {raw}") - } - } -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_parses_constrained_schema_payload(fixture: &LlamaFixture<'_>) -> Result<()> { - const NEGOTIATE_WITH_CAT_TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "negotiate_with_cat", - "description": "Attempt to negotiate with a cat. Outcomes are not guaranteed and may include the silent treatment.", - "parameters": { - "type": "object", - "properties": { - "topic": { - "type": "string", - "description": "What you are trying to negotiate, e.g. 'get off the keyboard' or 'stop knocking things off the table'" - }, - "bribe": { - "type": "string", - "enum": ["tuna", "salmon", "treats", "ear_scritches", "cardboard_box", "none"], - "description": "What you are offering in exchange" - }, - "desperation_level": { - "type": "integer", - "description": "How desperate you are, on a scale from 1 (mildly annoyed human) to 10 (it is 3am)", - "minimum": 1, - "maximum": 10 - } - }, - "required": ["topic"], - "additionalProperties": false - } - } - } -]"#; - - const NEGOTIATE_WITH_CAT_INPUT: &str = "\n\ -\n\ -\n\ -tuna\n\ -\n\ -\n\ -8\n\ -\n\ -\n\ -get off the keyboard\n\ -\n\ -\n\ -"; - - let outcome = ChatMessageParser::new(fixture.model, NEGOTIATE_WITH_CAT_TOOLS_JSON)? - .parse(NEGOTIATE_WITH_CAT_INPUT, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!( - "Qwen 3.5's tool-call payload must be parsed by the wrapper-side duck-type pass; \ - got Unrecognized" - ); - }; - - assert_eq!(parsed.tool_calls.len(), 1); - assert_eq!(parsed.tool_calls[0].name, "negotiate_with_cat"); - assert_eq!( - parsed.tool_calls[0].id, - ParsedToolCallId::NotProvidedByModel - ); - assert_eq!( - arguments_as_json(&parsed.tool_calls[0].arguments)?, - &json!({ - "bribe": "tuna", - "desperation_level": 8, - "topic": "get off the keyboard", - }), - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_parses_tool_call_payload(fixture: &LlamaFixture<'_>) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const QWEN_XML_PAYLOAD: &str = "\n\ -\n\ -\n\ -Paris\n\ -\n\ -\n\ -"; - - let outcome = - ChatMessageParser::new(fixture.model, TOOLS_JSON)?.parse(QWEN_XML_PAYLOAD, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!("expected Recognized for Qwen XML on a Qwen-3.5 model; got Unrecognized"); - }; - assert_eq!(parsed.tool_calls.len(), 1); - assert_eq!(parsed.tool_calls[0].name, "get_weather"); - let location = match &parsed.tool_calls[0].arguments { - ToolCallArguments::ValidJson(value) => value - .get("location") - .and_then(|value| value.as_str()) - .map(str::to_owned), - ToolCallArguments::InvalidJson(raw) => { - bail!("expected ValidJson, got InvalidJson: {raw}"); - } - }; - assert_eq!(location.as_deref(), Some("Paris")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_parses_partial_tool_call_returns_pending_state(fixture: &LlamaFixture<'_>) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const PARTIAL_QWEN_XML_PAYLOAD: &str = "\n\n) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const TWO_QWEN_XML_PAYLOADS: &str = "\n\ -\n\ -\n\ -Paris\n\ -\n\ -\n\ -\n\ -\n\ -\n\ -\n\ -Berlin\n\ -\n\ -\n\ -"; - - let outcome = - ChatMessageParser::new(fixture.model, TOOLS_JSON)?.parse(TWO_QWEN_XML_PAYLOADS, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!( - "expected Recognized for two Qwen XML payloads on a Qwen-3.5 model; got Unrecognized" - ); - }; - assert_eq!( - parsed.tool_calls, - vec![ - ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - "get_weather".to_owned(), - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ), - ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - "get_weather".to_owned(), - ToolCallArguments::ValidJson(json!({"location": "Berlin"})), - ), - ] - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_recognizes_empty_tool_calls_when_input_is_plain_content_with_tools_requested( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const PLAIN_CONTENT: &str = "Sorry, I cannot help with that."; - - let outcome = ChatMessageParser::new(fixture.model, TOOLS_JSON)?.parse(PLAIN_CONTENT, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!( - "Qwen 3.5 with tools requested + plain content must produce Recognized (with empty \ - tool_calls); got Unrecognized" - ); - }; - assert!( - parsed.tool_calls.is_empty(), - "expected no tool calls; got {:?}", - parsed.tool_calls - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128, -)] -fn qwen36_chat_inference_emits_reasoning_when_template_auto_opens( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let backend = fixture.backend; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let chat_template = model.chat_template(None)?; - let messages = vec![LlamaChatMessage::new( - "user".to_owned(), - "Hello! How are you?".to_owned(), - )?]; - let prompt = model.apply_chat_template(&chat_template, &messages, true, true)?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let tokens = model.str_to_token(&prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let prompt_token_count = u64::try_from(tokens.len())?; - - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; - - context.decode(&batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::greedy()?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: 512, - } - .run()?; - - assert_ne!(outcome.generated_raw, ""); - assert!(outcome.observed_reasoning > 0); - assert!(outcome.observed_content > 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(outcome.observed_tool_call, 0); - - let parsed = parse_partial_reasoning_response(model, &outcome.generated_raw)?; - assert_ne!(parsed.content, ""); - - let usage = classifier.into_usage(); - assert_eq!(usage.prompt_tokens, prompt_token_count); - assert_eq!(usage.reasoning_tokens, outcome.observed_reasoning); - assert_eq!(usage.undeterminable_tokens, 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn qwen36_classifier_does_not_emit_reasoning_for_thinking_disabled_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 200; - - const QWEN36_THINKING_DISABLED_PROMPT: &str = "\ -<|im_start|>user -What is 2 + 2?<|im_end|> -<|im_start|>assistant - - - - -"; - - const FORBIDDEN_MARKERS: &[&str] = &["", ""]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - QWEN36_THINKING_DISABLED_PROMPT, - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - context.decode(&batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - - assert_ne!(outcome.generated_raw, ""); - assert_eq!(outcome.observed_reasoning, 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(usage.reasoning_tokens, 0); - assert_eq!(usage.undeterminable_tokens, 0); - assert!(outcome.observed_content > 0); - assert_eq!(usage.completion_tokens(), outcome.observed_content); - - for forbidden in FORBIDDEN_MARKERS { - assert!(!outcome.content_stream.contains(forbidden)); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn qwen36_classifier_emits_reasoning_for_thinking_enabled_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 512; - - const QWEN36_THINKING_PROMPT: &str = "\ -<|im_start|>user -What is 2 + 2?<|im_end|> -<|im_start|>assistant - -"; - - const FORBIDDEN_MARKERS: &[&str] = &["", ""]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - QWEN36_THINKING_PROMPT, - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - context.decode(&batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - let parsed = parse_partial_reasoning_response(model, &outcome.generated_raw)?; - - assert_ne!(outcome.generated_raw, ""); - assert!(outcome.observed_reasoning > 0); - assert!(usage.reasoning_tokens > 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(usage.undeterminable_tokens, 0); - assert_eq!( - usage.completion_tokens(), - outcome.observed_content + outcome.observed_reasoning, - ); - - assert_ne!(parsed.reasoning_content, ""); - assert_eq!(outcome.reasoning_stream, parsed.reasoning_content); - assert_eq!(outcome.content_stream, parsed.content); - - for forbidden in FORBIDDEN_MARKERS { - assert!(!outcome.reasoning_stream.contains(forbidden)); - assert!(!outcome.content_stream.contains(forbidden)); - } - - Ok(()) -} - -fn visible_text_of_generation_ending_after( - model: &LlamaModel, - generated_text: &str, -) -> Result { - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut outcomes = Vec::new(); - - for token in model.str_to_token(generated_text, AddBos::Never, ParseSpecialTokens::Always)? { - assert_eq!( - classifier.ingest(token, &mut outcomes)?, - GenerationProgress::Continues - ); - } - - assert_eq!( - classifier.ingest( - model.token_eos().context("model declares an EOS token")?, - &mut outcomes - )?, - GenerationProgress::Ended - ); - - Ok(outcomes - .iter() - .map(|outcome| outcome.piece.visible()) - .collect()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_classifier_ends_generation_without_emitting_the_end_of_generation_token( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const GENERATED_TEXT: &str = "The answer is four."; - - assert_eq!( - visible_text_of_generation_ending_after(fixture.model, GENERATED_TEXT)?, - GENERATED_TEXT - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_classifier_releases_a_held_json_prefix_when_generation_ends( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const GENERATED_TEXT: &str = r#"{"answer": 4"#; - - assert_eq!( - visible_text_of_generation_ending_after(fixture.model, GENERATED_TEXT)?, - GENERATED_TEXT - ); - - Ok(()) -} diff --git a/llama-cpp-bindings-tests/tests/threadpool_detachment.rs b/llama-cpp-bindings-tests/tests/threadpool_detachment.rs new file mode 100644 index 000000000..d4f2cdd66 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/threadpool_detachment.rs @@ -0,0 +1,24 @@ +use anyhow::Result; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn detach_threadpool_completes_without_panic(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + context.detach_threadpool(); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/token_attributes.rs b/llama-cpp-bindings-tests/tests/token_attributes.rs new file mode 100644 index 000000000..dac76b5d3 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/token_attributes.rs @@ -0,0 +1,48 @@ +use anyhow::Context; +use anyhow::Result; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_attr_returns_attrs_for_bos(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let bos = model.token_bos().context("model declares a BOS token")?; + let attrs = model.token_attr(bos)?; + let bit_repr = format!("{:?}", *attrs); + assert!( + !bit_repr.is_empty(), + "token_attr(bos) must produce Debug output" + ); + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/token_candidates.rs b/llama-cpp-bindings-tests/tests/token_candidates.rs new file mode 100644 index 000000000..2b0645b72 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/token_candidates.rs @@ -0,0 +1,165 @@ +use anyhow::Result; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_bindings::token::data::LlamaTokenData; +use llama_cpp_bindings::token::data_array::LlamaTokenDataArray; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn token_data_array_has_entries_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let token_data_array = context.token_data_array()?; + + assert_ne!(token_data_array.data, Vec::::new()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn token_data_array_ith_returns_valid_data(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let last_index = i32::try_from(tokens.len() - 1)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let token_data_array = context.token_data_array_ith(last_index)?; + + assert_eq!( + token_data_array.data.len(), + usize::try_from(fixture.model.n_vocab())? + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn candidates_returns_n_vocab_entries(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let count = context.candidates()?.count(); + + assert_eq!(count, usize::try_from(fixture.model.n_vocab())?); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn candidates_ith_returns_n_vocab_entries(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let last_index = i32::try_from(tokens.len() - 1)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let count = context.candidates_ith(last_index)?.count(); + + assert_eq!(count, usize::try_from(fixture.model.n_vocab())?); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_refilled_candidate_array_matches_a_freshly_built_one( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let mut context = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let prompt_tokens = model.str_to_token("Hello", AddBos::Never, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + + batch.add_sequence(&prompt_tokens, 0, false)?; + context.decode(&batch)?; + + let last_index = batch.n_tokens() - 1; + let mut reused_candidates = LlamaTokenDataArray::new( + vec![LlamaTokenData::new(LlamaToken::new(7), 1.0, 0.5)], + true, + ); + reused_candidates.selected = Some(0); + + context.fill_token_data_array_ith(last_index, &mut reused_candidates)?; + + assert_eq!(reused_candidates, context.token_data_array_ith(last_index)?); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/tokenization.rs b/llama-cpp-bindings-tests/tests/tokenization.rs new file mode 100644 index 000000000..91d5f26c1 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/tokenization.rs @@ -0,0 +1,224 @@ +use anyhow::Result; +use llama_cpp_bindings::SampledToken; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn str_to_token_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let tokens = model.str_to_token("hello world", AddBos::Never, ParseSpecialTokens::Always)?; + assert_ne!(tokens, Vec::::new()); + let mut decoder = encoding_rs::UTF_8.new_decoder(); + let piece = + model.token_to_piece(&SampledToken::Content(tokens[0]), &mut decoder, false, None)?; + + assert_ne!(piece, ""); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn str_to_token_grows_buffer_when_initial_estimation_too_small( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let many_short_chars = "a b c d e f g h i j k l"; + let tokens = + fixture + .model + .str_to_token(many_short_chars, AddBos::Always, ParseSpecialTokens::Always)?; + + assert!( + tokens.len() > 8, + "expected regrow; got {} tokens", + tokens.len() + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn str_to_token_without_bos_yields_no_more_tokens_than_with_bos( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let tokens_with_bos = + model.str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let tokens_without_bos = + model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; + + assert!(tokens_with_bos.len() >= tokens_without_bos.len()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn str_to_token_with_many_tokens_triggers_buffer_resize(fixture: &LlamaFixture<'_>) -> Result<()> { + use std::fmt::Write; + + let many_numbers = (0..2000).fold(String::new(), |mut accumulator, number| { + let _ = write!(accumulator, "{number} "); + accumulator + }); + + let tokens = + fixture + .model + .str_to_token(&many_numbers, AddBos::Always, ParseSpecialTokens::Always)?; + + assert!(tokens.len() > many_numbers.len() / 2); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, +)] +fn tokenizing_without_special_token_parsing_keeps_control_markup_as_text( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let markup = "<|fim_prefix|>"; + let parsed = fixture + .model + .str_to_token(markup, AddBos::Never, ParseSpecialTokens::Always)?; + let plain = fixture + .model + .str_to_token(markup, AddBos::Never, ParseSpecialTokens::Never)?; + + assert_eq!(parsed.len(), 1); + assert!(plain.len() > 1); + assert!(!plain.contains(&parsed[0])); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/tokenize_synthetic.rs b/llama-cpp-bindings-tests/tests/tokenize_synthetic.rs new file mode 100644 index 000000000..8b8b6a087 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/tokenize_synthetic.rs @@ -0,0 +1,19 @@ +use anyhow::Result; +use llama_cpp_bindings::mtmd::MtmdBitmap; +use llama_cpp_bindings::mtmd::MtmdInputChunks; +use llama_cpp_bindings::mtmd::MtmdInputText; +use llama_cpp_test_harness::LlamaFixture; + +pub fn tokenize_synthetic(fixture: &LlamaFixture<'_>, prompt: &str) -> Result { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let image_data = vec![128u8; 64 * 64 * 3]; + let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; + let input_text = MtmdInputText { + text: prompt.to_owned(), + add_special: true, + parse_special: true, + }; + Ok(mtmd_ctx.tokenize(input_text, &[&bitmap])?) +} diff --git a/llama-cpp-bindings-tests/tests/tool_call_classification.rs b/llama-cpp-bindings-tests/tests/tool_call_classification.rs new file mode 100644 index 000000000..09561b892 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/tool_call_classification.rs @@ -0,0 +1,78 @@ +use anyhow::Result; +use llama_cpp_bindings::BareJsonToolCalls; +use llama_cpp_bindings::ChatOutputCompleteness; +use llama_cpp_bindings::SampledToken; +use llama_cpp_bindings::SampledTokenClassifier; +use llama_cpp_bindings::ingest_outcome::IngestOutcome; +use llama_cpp_bindings_tests::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; +use llama_cpp_bindings_tests::ingested_generation::ingested_generation; +use llama_cpp_bindings_tests::recognized_answer::recognized_answer; +use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +const REASONING_BEFORE_THE_CALL: &str = "Let me check the weather.\n\n\n"; +const QWEN_XML_PARIS_WEATHER_CALL: &str = "\n\ +\n\ +\n\ +Paris\n\ +\n\ +\n\ +"; + +fn raw_text_where(outcomes: &[IngestOutcome], belongs: impl Fn(&SampledToken) -> bool) -> String { + outcomes + .iter() + .filter(|outcome| belongs(&outcome.sampled_token)) + .map(|outcome| outcome.piece.raw()) + .collect() +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn qwen35_reasoning_closed_by_a_tool_call_is_attributed_to_each_section( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session( + fixture.model, + "What is the weather in Paris?", + GET_WEATHER_TOOLS_JSON, + true, + )?; + let answer = format!("{REASONING_BEFORE_THE_CALL}{QWEN_XML_PARIS_WEATHER_CALL}"); + let mut classifier = + SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let outcomes = ingested_generation(&mut classifier, fixture.model, &answer)?; + let parsed = recognized_answer(&session, &answer, ChatOutputCompleteness::Finished)?; + + assert_eq!( + raw_text_where(&outcomes, |token| matches!( + token, + SampledToken::Reasoning(_) + )), + REASONING_BEFORE_THE_CALL + ); + assert_eq!( + raw_text_where(&outcomes, |token| matches!( + token, + SampledToken::ToolCall(_) + )), + QWEN_XML_PARIS_WEATHER_CALL + ); + assert_eq!( + outcomes + .iter() + .filter(|outcome| matches!(outcome.sampled_token, SampledToken::Reasoning(_))) + .map(|outcome| outcome.piece.visible()) + .collect::(), + parsed.reasoning_content + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/tool_call_parsing.rs b/llama-cpp-bindings-tests/tests/tool_call_parsing.rs new file mode 100644 index 000000000..724ac869b --- /dev/null +++ b/llama-cpp-bindings-tests/tests/tool_call_parsing.rs @@ -0,0 +1,319 @@ +use anyhow::Result; +use llama_cpp_bindings::ChatAssistantTurn; +use llama_cpp_bindings::ChatOutputCompleteness; +use llama_cpp_bindings::ChatSession; +use llama_cpp_bindings::ChatSessionOptions; +use llama_cpp_bindings::ChatToolChoice; +use llama_cpp_bindings::ParsedToolCall; +use llama_cpp_bindings::ParsedToolCallId; +use llama_cpp_bindings::ToolCallArguments; +use llama_cpp_bindings::model::LlamaChatMessage; +use llama_cpp_bindings_tests::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; +use llama_cpp_bindings_tests::paris_weather_tool_call::paris_weather_tool_call; +use llama_cpp_bindings_tests::recognized_answer::recognized_answer; +use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; +use serde_json::json; + +const WEATHER_QUESTION: &str = "What is the weather in Paris?"; +const PLAIN_CONTENT_REFUSAL: &str = "Sorry, I cannot help with that."; + +const NEGOTIATE_WITH_CAT_TOOLS_JSON: &str = r#"[ + { + "type": "function", + "function": { + "name": "negotiate_with_cat", + "description": "Attempt to negotiate with a cat. Outcomes are not guaranteed and may include the silent treatment.", + "parameters": { + "type": "object", + "properties": { + "topic": { + "type": "string", + "description": "What you are trying to negotiate, e.g. 'get off the keyboard' or 'stop knocking things off the table'" + }, + "bribe": { + "type": "string", + "enum": ["tuna", "salmon", "treats", "ear_scritches", "cardboard_box", "none"], + "description": "What you are offering in exchange" + }, + "desperation_level": { + "type": "integer", + "description": "How desperate you are, on a scale from 1 (mildly annoyed human) to 10 (it is 3am)", + "minimum": 1, + "maximum": 10 + } + }, + "required": ["topic"], + "additionalProperties": false + } + } + } +]"#; + +const QWEN_XML_PARIS_WEATHER_CALL: &str = "\n\ +\n\ +\n\ +Paris\n\ +\n\ +\n\ +"; + +fn assert_parses_to_paris_weather_call(fixture: &LlamaFixture<'_>, answer: &str) -> Result<()> { + let session = user_turn_chat_session( + fixture.model, + WEATHER_QUESTION, + GET_WEATHER_TOOLS_JSON, + false, + )?; + let parsed = recognized_answer(&session, answer, ChatOutputCompleteness::Finished)?; + + assert_eq!(parsed.tool_calls, [paris_weather_tool_call()]); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn gemma4_parses_its_tool_call_format(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_parses_to_paris_weather_call( + fixture, + "<|tool_call>call:get_weather{location:<|\"|>Paris<|\"|>}", + ) +} + +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn glm47_parses_its_tool_call_format(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_parses_to_paris_weather_call( + fixture, + "get_weatherlocationParis", + ) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn mistral3_parses_its_tool_call_format(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_parses_to_paris_weather_call( + fixture, + r#"[TOOL_CALLS]get_weather[ARGS]{"location":"Paris"}"#, + ) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_parses_its_tool_call_format(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_parses_to_paris_weather_call(fixture, QWEN_XML_PARIS_WEATHER_CALL) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_parses_tool_call_arguments_into_their_schema_types( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session( + fixture.model, + "Get the cat off the keyboard.", + NEGOTIATE_WITH_CAT_TOOLS_JSON, + false, + )?; + let parsed = recognized_answer( + &session, + "\n\ +\n\ +\n\ +get off the keyboard\n\ +\n\ +\n\ +tuna\n\ +\n\ +\n\ +8\n\ +\n\ +\n\ +", + ChatOutputCompleteness::Finished, + )?; + + assert_eq!( + parsed.tool_calls, + [ParsedToolCall::new( + ParsedToolCallId::NotProvidedByModel, + "negotiate_with_cat".to_owned(), + ToolCallArguments::ValidJson(json!({ + "bribe": "tuna", + "desperation_level": 8, + "topic": "get off the keyboard", + })), + )] + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_parses_a_tool_call_cut_inside_its_arguments_as_a_named_call( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session( + fixture.model, + WEATHER_QUESTION, + GET_WEATHER_TOOLS_JSON, + false, + )?; + let parsed = recognized_answer( + &session, + "\n\n>(), + ["get_weather"] + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_parses_consecutive_tool_calls_in_order(fixture: &LlamaFixture<'_>) -> Result<()> { + let session = ChatSession::new( + fixture.model, + &[LlamaChatMessage::new( + "user".to_owned(), + "What is the weather in Paris and in Berlin?".to_owned(), + )?], + GET_WEATHER_TOOLS_JSON, + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::New, + enable_thinking: false, + parallel_tool_calls: true, + tool_choice: ChatToolChoice::Auto, + }, + )?; + let parsed = recognized_answer( + &session, + &format!( + "{QWEN_XML_PARIS_WEATHER_CALL}\n{}", + QWEN_XML_PARIS_WEATHER_CALL.replace("Paris", "Berlin") + ), + ChatOutputCompleteness::Finished, + )?; + + assert_eq!( + parsed.tool_calls, + [ + paris_weather_tool_call(), + ParsedToolCall::new( + ParsedToolCallId::NotProvidedByModel, + "get_weather".to_owned(), + ToolCallArguments::ValidJson(json!({"location": "Berlin"})), + ), + ] + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn plain_content_answering_a_tool_request_parses_without_tool_calls( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session( + fixture.model, + WEATHER_QUESTION, + GET_WEATHER_TOOLS_JSON, + false, + )?; + let parsed = recognized_answer( + &session, + PLAIN_CONTENT_REFUSAL, + ChatOutputCompleteness::Finished, + )?; + + assert_eq!(parsed.tool_calls, []); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn deepseek_r1_8b_plain_content_without_tools_parses_without_tool_calls( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, "Hello!", "[]", false)?; + let parsed = recognized_answer(&session, "Hello there.", ChatOutputCompleteness::Finished)?; + + assert_eq!(parsed.tool_calls, []); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs b/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs new file mode 100644 index 000000000..6afe72098 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs @@ -0,0 +1,214 @@ +use anyhow::Result; +use llama_cpp_bindings::GenerationProgress; +use llama_cpp_bindings::SampledToken; +use llama_cpp_bindings::SampledTokenClassifier; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn sample_returns_result_and_succeeds_with_valid_index(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let mut context = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let tokens = model.str_to_token("Hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + + batch.add_sequence(&tokens, 0, false)?; + + context.decode(&batch)?; + + let mut sampler = + LlamaSampler::chain_simple([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; + + let result = sampler.sample(&context, batch.n_tokens() - 1); + + assert!(result.is_ok()); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn sample_without_grammar_produces_multiple_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let mut context = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let prompt = + "<|im_start|>user\nSay hello<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; + let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + + batch.add_sequence(&tokens, 0, false)?; + + context.decode(&batch)?; + + let mut sampler = + LlamaSampler::chain_simple([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; + + let mut classifier = SampledTokenClassifier::undeterminable(model); + let mut outcomes = Vec::new(); + let mut sampled_count: u64 = 0; + + for (position, _) in (batch.n_tokens()..).zip(0..5) { + let turn = classifier.sample(&mut sampler, &context, -1, &mut outcomes)?; + + if turn.progress == GenerationProgress::Ended { + break; + } + + sampled_count += 1; + + let raw_as_sampled = SampledToken::Content(turn.token); + + batch.clear(); + batch.add(&raw_as_sampled, position, &[0], true)?; + + context.decode(&batch)?; + } + + classifier.finish(&mut outcomes)?; + + assert!( + sampled_count > 0, + "Should produce at least one token without grammar" + ); + let usage = classifier.into_usage(); + assert!( + usage.completion_tokens() >= sampled_count, + "completion_tokens ({}) must include the {sampled_count} non-EOG samples", + usage.completion_tokens() + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn apply_runs_sampler_over_token_data_array(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let tokens = fixture + .model + .str_to_token("Hi", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let mut data_array = context.token_data_array_ith(batch.n_tokens() - 1)?; + let sampler = LlamaSampler::greedy()?; + sampler.apply(&mut data_array)?; + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn sample_returns_token_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let tokens = fixture + .model + .str_to_token("Hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + let mut sampler = + LlamaSampler::chain_simple([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; + let result = sampler.sample(&context, batch.n_tokens() - 1); + + assert!(result.is_ok()); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/vocabulary_bounds.rs b/llama-cpp-bindings-tests/tests/vocabulary_bounds.rs new file mode 100644 index 000000000..064abfe2a --- /dev/null +++ b/llama-cpp-bindings-tests/tests/vocabulary_bounds.rs @@ -0,0 +1,61 @@ +use llama_cpp_bindings::TokenAttrError; +use llama_cpp_bindings::TokenOutsideVocabulary; +use llama_cpp_bindings::TokenToStringError; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_to_piece_bytes_rejects_tokens_outside_the_vocabulary(fixture: &LlamaFixture<'_>) { + let model = fixture.model; + let n_vocab = model.n_vocab(); + + assert_eq!( + [LlamaToken::new(-1), LlamaToken::new(n_vocab)] + .map(|token| model.token_to_piece_bytes(token, 8, true, None)), + [ + Err(TokenToStringError::TokenOutsideVocabulary( + TokenOutsideVocabulary { + token: LlamaToken::new(-1), + n_vocab, + } + )), + Err(TokenToStringError::TokenOutsideVocabulary( + TokenOutsideVocabulary { + token: LlamaToken::new(n_vocab), + n_vocab, + } + )), + ] + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_attr_rejects_tokens_outside_the_vocabulary(fixture: &LlamaFixture<'_>) { + let model = fixture.model; + let n_vocab = model.n_vocab(); + + assert_eq!( + model.token_attr(LlamaToken::new(n_vocab)).err(), + Some(TokenAttrError::TokenOutsideVocabulary( + TokenOutsideVocabulary { + token: LlamaToken::new(n_vocab), + n_vocab, + } + )) + ); +} diff --git a/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs b/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs new file mode 100644 index 000000000..3ae9c1a8b --- /dev/null +++ b/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs @@ -0,0 +1,92 @@ +use anyhow::Result; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn tokens_iterator_produces_valid_entries(fixture: &LlamaFixture<'_>) { + let model = fixture.model; + let mut count = 0; + + for (token, _piece_result) in model.tokens(false) { + assert!(token.0 >= 0); + count += 1; + + if count >= 100 { + break; + } + } + + assert_eq!(count, 100); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn n_vocab_matches_tokens_iterator_count(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let n_vocab = model.n_vocab(); + let count = model.tokens(false).count(); + + assert_eq!(count, usize::try_from(n_vocab)?); + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/vocabulary_type.rs b/llama-cpp-bindings-tests/tests/vocabulary_type.rs new file mode 100644 index 000000000..44c0e1c76 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/vocabulary_type.rs @@ -0,0 +1,43 @@ +use anyhow::Result; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn default_models_use_a_byte_pair_encoding_vocabulary(fixture: &LlamaFixture<'_>) -> Result<()> { + use llama_cpp_bindings::model::vocab_type::VocabType; + + assert_eq!(fixture.model.vocab_type()?, VocabType::Bpe); + + Ok(()) +} diff --git a/llama-cpp-bindings-types/src/lib.rs b/llama-cpp-bindings-types/src/lib.rs index 09f82ca00..79205641b 100644 --- a/llama-cpp-bindings-types/src/lib.rs +++ b/llama-cpp-bindings-types/src/lib.rs @@ -12,7 +12,6 @@ pub mod paired_quote_shape; pub mod parsed_chat_message; pub mod parsed_tool_call; pub mod parsed_tool_call_id; -pub mod reasoning_markers; pub mod token_usage; pub mod token_usage_error; pub mod tool_call_args_shape; @@ -30,7 +29,6 @@ pub use paired_quote_shape::PairedQuoteShape; pub use parsed_chat_message::ParsedChatMessage; pub use parsed_tool_call::ParsedToolCall; pub use parsed_tool_call_id::ParsedToolCallId; -pub use reasoning_markers::ReasoningMarkers; pub use token_usage::TokenUsage; pub use token_usage_error::TokenUsageError; pub use tool_call_args_shape::ToolCallArgsShape; diff --git a/llama-cpp-bindings-types/src/reasoning_markers.rs b/llama-cpp-bindings-types/src/reasoning_markers.rs deleted file mode 100644 index b29636be8..000000000 --- a/llama-cpp-bindings-types/src/reasoning_markers.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct ReasoningMarkers { - pub open: String, - pub closes: Vec, -} diff --git a/llama-cpp-bindings/src/bare_json_tool_call_probe.rs b/llama-cpp-bindings/src/bare_json_tool_call_probe.rs new file mode 100644 index 000000000..b7b5bd348 --- /dev/null +++ b/llama-cpp-bindings/src/bare_json_tool_call_probe.rs @@ -0,0 +1,210 @@ +use std::mem; + +use crate::ingest_outcome::IngestOutcome; +use crate::json_probe_outcome::JsonProbeOutcome; +use crate::sampled_token::SampledToken; +use crate::streaming_json_probe::StreamingJsonProbe; +use crate::token_piece::TokenPiece; + +#[derive(Clone, Debug, Eq, PartialEq)] +enum ProbeState { + Idle, + Holding { + probe: StreamingJsonProbe, + held: Vec, + }, +} + +const fn is_visible_content(outcome: &IngestOutcome) -> bool { + matches!(outcome.sampled_token, SampledToken::Content(_)) + && matches!(outcome.piece, TokenPiece::Visible(_)) +} + +/// Holds visible content that opens a bare JSON object until the object settles. +/// +/// A complete `{"name": ..., "arguments": ...}` object is released as tool-call tokens; anything +/// else is released as the content it was. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct BareJsonToolCallProbe { + state: ProbeState, +} + +impl Default for BareJsonToolCallProbe { + fn default() -> Self { + Self { + state: ProbeState::Idle, + } + } +} + +impl BareJsonToolCallProbe { + pub fn pass(&mut self, outcome: IngestOutcome, released: &mut Vec) { + let is_content = is_visible_content(&outcome); + + match &mut self.state { + ProbeState::Holding { .. } if !is_content => { + self.release(released); + released.push(outcome); + } + ProbeState::Holding { probe, held } => { + let probe_outcome = probe.feed(outcome.piece.raw()); + held.push(outcome); + self.settle(probe_outcome, released); + } + ProbeState::Idle if is_content && outcome.piece.raw().trim_start().starts_with('{') => { + let mut probe = StreamingJsonProbe::default(); + let probe_outcome = probe.feed(outcome.piece.raw()); + + self.state = ProbeState::Holding { + probe, + held: vec![outcome], + }; + self.settle(probe_outcome, released); + } + ProbeState::Idle => released.push(outcome), + } + } + + fn settle(&mut self, probe_outcome: JsonProbeOutcome, released: &mut Vec) { + match probe_outcome { + JsonProbeOutcome::StillPossiblyValid => {} + JsonProbeOutcome::CompletedValid => self.commit_as_tool_call(released), + JsonProbeOutcome::Failed => self.release(released), + } + } + + fn commit_as_tool_call(&mut self, released: &mut Vec) { + if let ProbeState::Holding { held, .. } = mem::replace(&mut self.state, ProbeState::Idle) { + released.extend(held.into_iter().map(|outcome| IngestOutcome { + sampled_token: SampledToken::ToolCall(outcome.sampled_token.token()), + piece: outcome.piece, + })); + } + } + + /// Releases every held token as the content it was. + pub fn release(&mut self, released: &mut Vec) { + if let ProbeState::Holding { held, .. } = mem::replace(&mut self.state, ProbeState::Idle) { + released.extend(held); + } + } +} + +#[cfg(test)] +mod tests { + use super::BareJsonToolCallProbe; + use crate::ingest_outcome::IngestOutcome; + use crate::sampled_token::SampledToken; + use crate::token::LlamaToken; + use crate::token_piece::TokenPiece; + + fn content(piece: &str) -> IngestOutcome { + IngestOutcome { + sampled_token: SampledToken::Content(LlamaToken::new(1)), + piece: TokenPiece::Visible(piece.to_owned()), + } + } + + fn tool_call(piece: &str) -> IngestOutcome { + IngestOutcome { + sampled_token: SampledToken::ToolCall(LlamaToken::new(1)), + piece: TokenPiece::Visible(piece.to_owned()), + } + } + + fn pass_all( + probe: &mut BareJsonToolCallProbe, + outcomes: Vec, + ) -> Vec { + let mut released = Vec::new(); + + for outcome in outcomes { + probe.pass(outcome, &mut released); + } + + released + } + + #[test] + fn content_that_does_not_open_an_object_passes_through() { + let mut probe = BareJsonToolCallProbe::default(); + + assert_eq!(pass_all(&mut probe, vec![content("Hi")]), [content("Hi")]); + } + + #[test] + fn a_complete_named_tool_call_object_is_released_as_tool_call_tokens() { + let mut probe = BareJsonToolCallProbe::default(); + + assert_eq!( + pass_all( + &mut probe, + vec![ + content(" {\"name\":"), + content("\"f\",\"arguments\""), + content(":{}}") + ] + ), + [ + tool_call(" {\"name\":"), + tool_call("\"f\",\"arguments\""), + tool_call(":{}}"), + ] + ); + } + + #[test] + fn an_object_that_is_not_a_tool_call_is_released_as_content() { + let mut probe = BareJsonToolCallProbe::default(); + + assert_eq!( + pass_all(&mut probe, vec![content("{\"foo\":"), content("\"bar\"}")]), + [content("{\"foo\":"), content("\"bar\"}")] + ); + } + + #[test] + fn an_unfinished_object_is_held_until_released() { + let mut probe = BareJsonToolCallProbe::default(); + let mut released = pass_all(&mut probe, vec![content("{\"name\":")]); + + assert_eq!(released, []); + + probe.release(&mut released); + + assert_eq!(released, [content("{\"name\":")]); + } + + #[test] + fn a_token_outside_content_releases_the_held_object_before_itself() { + let mut probe = BareJsonToolCallProbe::default(); + let reasoning = IngestOutcome { + sampled_token: SampledToken::Reasoning(LlamaToken::new(2)), + piece: TokenPiece::Visible("hmm".to_owned()), + }; + + assert_eq!( + pass_all(&mut probe, vec![content("{\"name\":"), reasoning.clone()]), + [content("{\"name\":"), reasoning] + ); + } + + #[test] + fn two_consecutive_tool_call_objects_are_recognized_separately() { + let mut probe = BareJsonToolCallProbe::default(); + + assert_eq!( + pass_all( + &mut probe, + vec![ + content("{\"name\":\"f\",\"arguments\":{}}"), + content("{\"name\":\"g\",\"arguments\":{}}"), + ] + ), + [ + tool_call("{\"name\":\"f\",\"arguments\":{}}"), + tool_call("{\"name\":\"g\",\"arguments\":{}}"), + ] + ); + } +} diff --git a/llama-cpp-bindings/src/chat_assistant_prefill.rs b/llama-cpp-bindings/src/chat_assistant_prefill.rs new file mode 100644 index 000000000..1c2fa42e1 --- /dev/null +++ b/llama-cpp-bindings/src/chat_assistant_prefill.rs @@ -0,0 +1,7 @@ +/// The beginning of an assistant answer the model continues: its reasoning, closed, and the +/// content written so far. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ChatAssistantPrefill { + pub reasoning_content: String, + pub content: String, +} diff --git a/llama-cpp-bindings/src/chat_assistant_turn.rs b/llama-cpp-bindings/src/chat_assistant_turn.rs new file mode 100644 index 000000000..8a7c6602f --- /dev/null +++ b/llama-cpp-bindings/src/chat_assistant_turn.rs @@ -0,0 +1,34 @@ +use crate::chat_assistant_prefill::ChatAssistantPrefill; + +/// How the rendered prompt leaves the assistant's turn, as llama.cpp's `add_generation_prompt` +/// and `continue_final_message` do. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ChatAssistantTurn { + /// The prompt ends after the last message. + Omitted, + /// The prompt opens a new assistant turn. + New, + /// The prompt opens an assistant turn whose reasoning starts with the given text and is left + /// open for the model to continue. + ContinueReasoning(String), + /// The prompt opens an assistant turn with the given reasoning closed and content left open + /// for the model to continue. + ContinueContent(ChatAssistantPrefill), +} + +impl From<&ChatAssistantTurn> for llama_cpp_bindings_sys::llama_rs_chat_assistant_turn { + fn from(assistant_turn: &ChatAssistantTurn) -> Self { + match assistant_turn { + ChatAssistantTurn::Omitted => { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_ASSISTANT_TURN_OMITTED + } + ChatAssistantTurn::New => llama_cpp_bindings_sys::LLAMA_RS_CHAT_ASSISTANT_TURN_NEW, + ChatAssistantTurn::ContinueReasoning(_) => { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_ASSISTANT_TURN_CONTINUE_REASONING + } + ChatAssistantTurn::ContinueContent(_) => { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_ASSISTANT_TURN_CONTINUE_CONTENT + } + } + } +} diff --git a/llama-cpp-bindings/src/chat_classification.rs b/llama-cpp-bindings/src/chat_classification.rs new file mode 100644 index 000000000..9247e210b --- /dev/null +++ b/llama-cpp-bindings/src/chat_classification.rs @@ -0,0 +1,7 @@ +use crate::chat_spans::ChatSpans; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ChatClassification { + Spans(ChatSpans), + OutputDoesNotMatchFormat, +} diff --git a/llama-cpp-bindings/src/chat_grammar_trigger.rs b/llama-cpp-bindings/src/chat_grammar_trigger.rs new file mode 100644 index 000000000..51a3e6d3c --- /dev/null +++ b/llama-cpp-bindings/src/chat_grammar_trigger.rs @@ -0,0 +1,9 @@ +use crate::token::LlamaToken; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ChatGrammarTrigger { + Token(LlamaToken), + Word(String), + Pattern(String), + PatternFull(String), +} diff --git a/llama-cpp-bindings/src/chat_message_parse_outcome.rs b/llama-cpp-bindings/src/chat_message_parse_outcome.rs index 833fbc596..845a90e39 100644 --- a/llama-cpp-bindings/src/chat_message_parse_outcome.rs +++ b/llama-cpp-bindings/src/chat_message_parse_outcome.rs @@ -7,52 +7,3 @@ pub enum ChatMessageParseOutcome { Recognized(ParsedChatMessage), Unrecognized(RawChatMessage), } - -#[cfg(test)] -mod tests { - use llama_cpp_bindings_types::ParsedChatMessage; - use llama_cpp_bindings_types::ParsedToolCall; - - use super::ChatMessageParseOutcome; - use crate::raw_chat_message::RawChatMessage; - - #[test] - fn both_variants_destructure_to_their_inner_payloads() { - let outcomes = [ - ChatMessageParseOutcome::Recognized(ParsedChatMessage::new( - "content".to_owned(), - "reasoning".to_owned(), - Vec::new(), - )), - ChatMessageParseOutcome::Unrecognized(RawChatMessage { - text: "raw input".to_owned(), - is_partial: false, - ffi_error_message: "parser bailed".to_owned(), - }), - ]; - - let mut saw_recognized = false; - let mut saw_unrecognized = false; - for outcome in outcomes { - match outcome { - ChatMessageParseOutcome::Recognized(parsed) => { - assert_eq!(parsed.content, "content"); - assert_eq!(parsed.reasoning_content, "reasoning"); - assert_eq!(parsed.tool_calls, Vec::::new()); - saw_recognized = true; - } - ChatMessageParseOutcome::Unrecognized(raw) => { - assert_eq!(raw.text, "raw input"); - assert!(!raw.is_partial); - assert_eq!(raw.ffi_error_message, "parser bailed"); - saw_unrecognized = true; - } - } - } - - assert!( - saw_recognized && saw_unrecognized, - "both variants must dispatch through the match" - ); - } -} diff --git a/llama-cpp-bindings/src/chat_message_parser.rs b/llama-cpp-bindings/src/chat_message_parser.rs deleted file mode 100644 index 81bb36439..000000000 --- a/llama-cpp-bindings/src/chat_message_parser.rs +++ /dev/null @@ -1,2633 +0,0 @@ -use std::ffi::CStr; -use std::ffi::CString; -use std::ffi::c_char; -use std::ptr; -use std::ptr::NonNull; - -use llama_cpp_bindings_types::ModelProvidedToolCallId; -use llama_cpp_bindings_types::ModelProvidedToolCallIdError; -use llama_cpp_bindings_types::ParsedChatMessage; -use llama_cpp_bindings_types::ParsedToolCall; -use llama_cpp_bindings_types::ParsedToolCallId; -use llama_cpp_bindings_types::ReasoningMarkers; -use llama_cpp_bindings_types::ToolCallArguments; -use llama_cpp_ffi_status::read_and_free_cpp_string; - -use crate::chat_message_parse_outcome::ChatMessageParseOutcome; -use crate::chat_template_tool_calls; -use crate::error::parse_chat_message_error::ParseChatMessageError; -use crate::model::LlamaModel; -use crate::raw_chat_message::RawChatMessage; -use crate::tool_call_format; -use crate::tool_call_format::ToolCallFormatOutcome; - -/// # Safety -/// -/// `free_error` must be the pointer populated by the preceding -/// `llama_rs_parsed_chat_free` call, or null. The destructor-threw arm reads and -/// frees it. -unsafe fn parsed_chat_free_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_parsed_chat_free_status, - free_error: *mut c_char, -) -> Result<(), ParseChatMessageError> { - match status { - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_ERROR_STRING_ALLOCATION_FAILED => { - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - free_error, - "llama_rs_parsed_chat_free", - "reported a thrown C++ exception without an error message", - ) - }?; - - Err(ParseChatMessageError::DestructorFailed { message }) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_free", - code: i64::from(other), - } - .into()), - } -} - -/// # Safety -/// -/// `out_error` must reference the pointer populated by a `llama_rs_parse_chat_message` call -/// that reported a thrown C++ exception. The error is read, freed, and the referenced pointer -/// is nulled so the later free in the caller does not double-free. -unsafe fn thrown_parse_exception_error( - out_error: *mut *mut c_char, - error_from_message: fn(String) -> ParseChatMessageError, -) -> ParseChatMessageError { - match unsafe { - read_and_free_cpp_string( - *out_error, - "llama_rs_parse_chat_message", - "reported a thrown C++ exception without an error message", - ) - } { - Ok(message) => { - unsafe { *out_error = ptr::null_mut() }; - - error_from_message(message) - } - Err(missing_message) => missing_message.into(), - } -} - -/// # Safety -/// -/// `handle` must be the parsed-chat handle (or null) and `out_error` must reference the -/// pointer populated by the preceding `llama_rs_parse_chat_message` call. In the CXX-exception -/// arms the error is read, freed, and the referenced pointer is nulled so the later free in the -/// caller does not double-free. -unsafe fn parse_chat_message_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_parse_chat_message_status, - handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, - out_error: *mut *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_OK => { - if handle.is_null() { - Err(crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "success status contained a null parsed-chat handle", - } - .into()) - } else { - collect_parsed_chat_message(handle) - } - } - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_ERROR_STRING_ALLOCATION_FAILED => { - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_LLAMA_CPP_THREW_CXX_EXCEPTION => { - Err(unsafe { - thrown_parse_exception_error(out_error, |message| { - ParseChatMessageError::MessageUnrecognized { message } - }) - }) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_TOOLS_PARSER_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "was given a null tools parser argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_INPUT_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "was given a null input argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_OUT_HANDLE_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "was given a null out_handle argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "was given a null out_error argument", - } - .into()) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_parse_chat_message", - code: i64::from(other), - } - .into()), - } -} - -fn outcome_from_via_ffi_result( - via_ffi_result: Result, - input: &str, - is_partial: bool, -) -> Result { - match via_ffi_result { - Ok(parsed) => Ok(ChatMessageParseOutcome::Recognized(parsed)), - Err(ParseChatMessageError::MessageUnrecognized { message }) => { - Ok(ChatMessageParseOutcome::Unrecognized(RawChatMessage { - text: input.to_owned(), - is_partial, - ffi_error_message: message, - })) - } - Err(other) => Err(other), - } -} - -fn collect_parsed_chat_message( - handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, -) -> Result { - if handle.is_null() { - return Ok(ParsedChatMessage::default()); - } - - let content = read_parsed_chat_content(handle)?; - let reasoning_content = read_parsed_chat_reasoning_content(handle)?; - let count = read_parsed_chat_tool_call_count(handle)?; - - let mut tool_calls = Vec::with_capacity(count); - for index in 0..count { - let id = tool_call_id_reported_by_llama_cpp(read_parsed_chat_tool_call_id(handle, index)?); - let name = read_parsed_chat_tool_call_name(handle, index)?; - let arguments_json = read_parsed_chat_tool_call_arguments(handle, index)?; - - let arguments = ToolCallArguments::from_string(arguments_json); - tool_calls.push(ParsedToolCall::new(id, name, arguments)); - } - - Ok(ParsedChatMessage::new( - content, - reasoning_content, - tool_calls, - )) -} - -/// # Safety -/// -/// `out_string` and `out_error` must be the pointers populated by the preceding -/// `llama_rs_parsed_chat_content` call (or null when no value/error was produced); each is -/// read and freed in exactly one match arm. -unsafe fn parsed_chat_content_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_parsed_chat_content_status, - out_string: *mut c_char, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_OK => { - consume_accessor_string(out_string, "llama_rs_parsed_chat_content") - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_ERROR_STRING_ALLOCATION_FAILED => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_LLAMA_CPP_OUT_OF_MEMORY => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - out_error, - "llama_rs_parsed_chat_content", - "reported a thrown C++ exception without an error message", - ) - }?; - Err(ParseChatMessageError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_NULL_HANDLE_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_content", - detail: "was given a null handle argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_NULL_OUT_STRING_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_content", - detail: "was given a null out_string argument", - } - .into()) - } - other => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_content", - code: i64::from(other), - } - .into()) - } - } -} - -fn read_parsed_chat_content( - handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, -) -> Result { - let mut out_string: *mut c_char = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_parsed_chat_content( - handle, - &raw mut out_string, - &raw mut out_error, - ) - }; - unsafe { parsed_chat_content_status_to_result(status, out_string, out_error) } -} - -/// # Safety -/// -/// `out_string` and `out_error` must be the pointers populated by the preceding -/// `llama_rs_parsed_chat_reasoning_content` call (or null when no value/error was produced); -/// each is read and freed in exactly one match arm. -unsafe fn parsed_chat_reasoning_content_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_parsed_chat_reasoning_content_status, - out_string: *mut c_char, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_OK => { - consume_accessor_string(out_string, "llama_rs_parsed_chat_reasoning_content") - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_ERROR_STRING_ALLOCATION_FAILED => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_LLAMA_CPP_OUT_OF_MEMORY => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = - unsafe { read_and_free_cpp_string(out_error, "llama_rs_parsed_chat_reasoning_content", "reported a thrown C++ exception without an error message") }?; - Err(ParseChatMessageError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_NULL_HANDLE_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_reasoning_content", - detail: "was given a null handle argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_NULL_OUT_STRING_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_reasoning_content", - detail: "was given a null out_string argument", - } - .into()) - } - other => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_reasoning_content", - code: i64::from(other), - } - .into()) - } - } -} - -fn read_parsed_chat_reasoning_content( - handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, -) -> Result { - let mut out_string: *mut c_char = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_parsed_chat_reasoning_content( - handle, - &raw mut out_string, - &raw mut out_error, - ) - }; - unsafe { parsed_chat_reasoning_content_status_to_result(status, out_string, out_error) } -} - -/// # Safety -/// -/// `out_error` must be the pointer populated by the preceding -/// `llama_rs_parsed_chat_tool_call_count` call (or null when no error was produced); it is -/// freed in exactly one match arm. -unsafe fn parsed_chat_tool_call_count_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_count_status, - out_count: usize, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_OK => Ok(out_count), - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_ERROR_STRING_ALLOCATION_FAILED => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_LLAMA_CPP_OUT_OF_MEMORY => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = - unsafe { read_and_free_cpp_string(out_error, "llama_rs_parsed_chat_tool_call_count", "reported a thrown C++ exception without an error message") }?; - Err(ParseChatMessageError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_NULL_HANDLE_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_count", - detail: "was given a null handle argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_NULL_OUT_COUNT_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_count", - detail: "was given a null out_count argument", - } - .into()) - } - other => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_tool_call_count", - code: i64::from(other), - } - .into()) - } - } -} - -fn read_parsed_chat_tool_call_count( - handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, -) -> Result { - let mut out_count: usize = 0; - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_count( - handle, - &raw mut out_count, - &raw mut out_error, - ) - }; - unsafe { parsed_chat_tool_call_count_status_to_result(status, out_count, out_error) } -} - -/// # Safety -/// -/// `out_string` and `out_error` must be the pointers populated by the preceding -/// `llama_rs_parsed_chat_tool_call_id` call (or null when no value/error was produced); each -/// is read and freed in exactly one match arm. -unsafe fn parsed_chat_tool_call_id_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_id_status, - index: usize, - out_string: *mut c_char, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_OK => { - consume_accessor_string(out_string, "llama_rs_parsed_chat_tool_call_id") - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_INDEX_OUT_OF_BOUNDS => { - Err(ParseChatMessageError::ToolCallIdIndexOutOfBounds { index }) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_ERROR_STRING_ALLOCATION_FAILED => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_LLAMA_CPP_OUT_OF_MEMORY => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = - unsafe { read_and_free_cpp_string(out_error, "llama_rs_parsed_chat_tool_call_id", "reported a thrown C++ exception without an error message") }?; - Err(ParseChatMessageError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_NULL_HANDLE_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_id", - detail: "was given a null handle argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_NULL_OUT_STRING_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_id", - detail: "was given a null out_string argument", - } - .into()) - } - other => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_tool_call_id", - code: i64::from(other), - } - .into()) - } - } -} - -fn read_parsed_chat_tool_call_id( - handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, - index: usize, -) -> Result { - let mut out_string: *mut c_char = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_id( - handle, - index, - &raw mut out_string, - &raw mut out_error, - ) - }; - unsafe { parsed_chat_tool_call_id_status_to_result(status, index, out_string, out_error) } -} - -/// # Safety -/// -/// `out_string` and `out_error` must be the pointers populated by the preceding -/// `llama_rs_parsed_chat_tool_call_name` call (or null when no value/error was produced); each -/// is read and freed in exactly one match arm. -unsafe fn parsed_chat_tool_call_name_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_name_status, - index: usize, - out_string: *mut c_char, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_OK => { - consume_accessor_string(out_string, "llama_rs_parsed_chat_tool_call_name") - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_INDEX_OUT_OF_BOUNDS => { - Err(ParseChatMessageError::ToolCallNameIndexOutOfBounds { index }) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_ERROR_STRING_ALLOCATION_FAILED => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_LLAMA_CPP_OUT_OF_MEMORY => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = - unsafe { read_and_free_cpp_string(out_error, "llama_rs_parsed_chat_tool_call_name", "reported a thrown C++ exception without an error message") }?; - Err(ParseChatMessageError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_NULL_HANDLE_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_name", - detail: "was given a null handle argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_NULL_OUT_STRING_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_name", - detail: "was given a null out_string argument", - } - .into()) - } - other => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_tool_call_name", - code: i64::from(other), - } - .into()) - } - } -} - -fn read_parsed_chat_tool_call_name( - handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, - index: usize, -) -> Result { - let mut out_string: *mut c_char = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_name( - handle, - index, - &raw mut out_string, - &raw mut out_error, - ) - }; - unsafe { parsed_chat_tool_call_name_status_to_result(status, index, out_string, out_error) } -} - -/// # Safety -/// -/// `out_string` and `out_error` must be the pointers populated by the preceding -/// `llama_rs_parsed_chat_tool_call_arguments` call (or null when no value/error was produced); -/// each is read and freed in exactly one match arm. -unsafe fn parsed_chat_tool_call_arguments_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_arguments_status, - index: usize, - out_string: *mut c_char, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_OK => { - consume_accessor_string(out_string, "llama_rs_parsed_chat_tool_call_arguments") - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_INDEX_OUT_OF_BOUNDS => { - Err(ParseChatMessageError::ToolCallArgumentsIndexOutOfBounds { index }) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_ERROR_STRING_ALLOCATION_FAILED => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_LLAMA_CPP_OUT_OF_MEMORY => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = - unsafe { read_and_free_cpp_string(out_error, "llama_rs_parsed_chat_tool_call_arguments", "reported a thrown C++ exception without an error message") }?; - Err(ParseChatMessageError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NULL_HANDLE_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_arguments", - detail: "was given a null handle argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NULL_OUT_STRING_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_arguments", - detail: "was given a null out_string argument", - } - .into()) - } - other => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_tool_call_arguments", - code: i64::from(other), - } - .into()) - } - } -} - -fn read_parsed_chat_tool_call_arguments( - handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, - index: usize, -) -> Result { - let mut out_string: *mut c_char = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_arguments( - handle, - index, - &raw mut out_string, - &raw mut out_error, - ) - }; - unsafe { - parsed_chat_tool_call_arguments_status_to_result(status, index, out_string, out_error) - } -} - -fn consume_accessor_string( - ptr: *mut c_char, - operation: &'static str, -) -> Result { - if ptr.is_null() { - return Err(crate::FfiContractError { - operation, - detail: "success status contained a null string", - } - .into()); - } - let bytes = unsafe { CStr::from_ptr(ptr) }.to_bytes().to_vec(); - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(ptr) }; - Ok(String::from_utf8(bytes)?) -} - -struct ReasoningSplit { - reasoning: String, - content: String, -} - -fn restore_partial_reasoning( - parsed: &mut ParsedChatMessage, - input: &str, - reasoning_markers: Option<&ReasoningMarkers>, - is_partial: bool, -) { - if !is_partial { - return; - } - if reasoning_markers.is_some_and(|markers| input.contains(&markers.open)) { - let split = split_reasoning_prefix(input, reasoning_markers, None, true); - parsed.reasoning_content = split.reasoning; - parsed.content = split.content; - return; - } - if let Some(open) = reasoning_markers.map(|markers| markers.open.trim()) - && let Some(reasoning) = parsed.reasoning_content.trim_start().strip_prefix(open) - { - parsed.reasoning_content = reasoning.to_owned(); - } -} - -fn split_reasoning_prefix( - input: &str, - reasoning_markers: Option<&ReasoningMarkers>, - tool_call_open: Option<&str>, - is_partial: bool, -) -> ReasoningSplit { - let content_only = || ReasoningSplit { - reasoning: String::new(), - content: prefix_before_optional(input, tool_call_open), - }; - - let Some(reasoning_markers) = reasoning_markers else { - return content_only(); - }; - let Some(open_pos) = input.find(&reasoning_markers.open) else { - return content_only(); - }; - - let after_open = &input[open_pos + reasoning_markers.open.len()..]; - let closing_marker = reasoning_markers - .closes - .iter() - .enumerate() - .filter_map(|(marker_index, marker)| { - after_open - .find(marker) - .map(|offset| (offset, marker_index, marker)) - }) - .min_by_key(|(offset, marker_index, _)| (*offset, *marker_index)); - let Some((close_offset, _, close_marker)) = closing_marker else { - return if is_partial { - ReasoningSplit { - reasoning: prefix_before_optional(after_open, tool_call_open), - content: input[..open_pos].to_owned(), - } - } else { - content_only() - }; - }; - - let reasoning = after_open[..close_offset].to_owned(); - let after_close = &after_open[close_offset + close_marker.len()..]; - - ReasoningSplit { - reasoning, - content: prefix_before_optional(after_close, tool_call_open), - } -} - -fn prefix_before_optional(text: &str, marker: Option<&str>) -> String { - marker.map_or_else( - || text.to_owned(), - |marker| { - text.find(marker) - .map_or_else(|| text.to_owned(), |pos| text[..pos].to_owned()) - }, - ) -} - -fn tool_call_id_reported_by_llama_cpp(llama_cpp_tool_call_id: String) -> ParsedToolCallId { - match ModelProvidedToolCallId::try_from(llama_cpp_tool_call_id) { - Ok(model_provided_tool_call_id) => { - ParsedToolCallId::ModelProvided(model_provided_tool_call_id) - } - Err(ModelProvidedToolCallIdError::Empty) => ParsedToolCallId::NotProvidedByModel, - } -} - -/// # Safety -/// -/// `out_error` must reference the pointer populated by the preceding -/// `llama_rs_chat_tools_parser_create` call (or null); it is read, freed, and nulled only in -/// the CXX-exception arm. `tools_parser` must be the pointer populated by the same call. -unsafe fn chat_tools_parser_create_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_chat_tools_parser_create_status, - tools_parser: *mut llama_cpp_bindings_sys::llama_rs_chat_tools_parser, - out_error: *mut *mut c_char, -) -> Result, ParseChatMessageError> { - match status { - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_OK => NonNull::new(tools_parser) - .ok_or_else(|| { - crate::FfiContractError { - operation: "llama_rs_chat_tools_parser_create", - detail: "success status contained a null tools parser handle", - } - .into() - }), - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_TOOLS_NOT_AN_ARRAY => { - Err(ParseChatMessageError::ToolsNotAnArray) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_ERROR_STRING_ALLOCATION_FAILED => { - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - *out_error, - "llama_rs_chat_tools_parser_create", - "reported a thrown C++ exception without an error message", - ) - }?; - - unsafe { *out_error = ptr::null_mut() }; - - Err(ParseChatMessageError::ToolsParserBuildFailed { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_PARSER_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_chat_tools_parser_create", - detail: "was given a null parser argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_TOOLS_JSON_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_chat_tools_parser_create", - detail: "was given a null tools_json argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_OUT_TOOLS_PARSER_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_chat_tools_parser_create", - detail: "was given a null out_tools_parser argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_chat_tools_parser_create", - detail: "was given a null out_error argument", - } - .into()) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_chat_tools_parser_create", - code: i64::from(other), - } - .into()), - } -} - -/// # Safety -/// -/// `out_error` must be the pointer populated by the preceding -/// `llama_rs_chat_tools_parser_free` call, or null. The destructor-threw arm reads and frees it. -unsafe fn chat_tools_parser_free_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_chat_tools_parser_free_status, - out_error: *mut c_char, -) -> Result<(), ParseChatMessageError> { - match status { - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_FREE_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_FREE_ERROR_STRING_ALLOCATION_FAILED => { - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_FREE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - out_error, - "llama_rs_chat_tools_parser_free", - "reported a thrown C++ exception without an error message", - ) - }?; - - Err(ParseChatMessageError::DestructorFailed { message }) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_chat_tools_parser_free", - code: i64::from(other), - } - .into()), - } -} - -/// Parses generated chat messages against one set of tools, building the -/// model-specific tools parser once for every message it parses. -pub struct ChatMessageParser { - reasoning_markers: Option, - tools_parser: NonNull, -} - -unsafe impl Send for ChatMessageParser {} - -impl ChatMessageParser { - /// # Errors - /// - /// Returns [`ParseChatMessageError`] when reasoning-marker detection fails, the model has - /// no chat parser, `tools_json` contains a NUL byte or is not a JSON array, or the tools - /// parser cannot be built. - pub fn new(model: &LlamaModel, tools_json: &str) -> Result { - let reasoning_markers = model.reasoning_markers()?.cloned(); - let chat_parser = model.chat_parser_ptr()?; - let tools_json_cstring = - CString::new(tools_json).map_err(ParseChatMessageError::ToolsContainNulByte)?; - let mut out_tools_parser: *mut llama_cpp_bindings_sys::llama_rs_chat_tools_parser = - ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_chat_tools_parser_create( - chat_parser, - tools_json_cstring.as_ptr(), - &raw mut out_tools_parser, - &raw mut out_error, - ) - }; - - let tools_parser = unsafe { - chat_tools_parser_create_status_to_result(status, out_tools_parser, &raw mut out_error) - }; - - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - - Ok(Self { - reasoning_markers, - tools_parser: tools_parser?, - }) - } - - /// # Errors - /// - /// Returns [`ParseChatMessageError`] when `input` contains a NUL byte, the FFI returns a - /// non-OK status other than a message parse exception, or accessor strings are not valid - /// UTF-8. - pub fn parse( - &self, - input: &str, - is_partial: bool, - ) -> Result { - let reasoning_markers = self.reasoning_markers.as_ref(); - - for candidate in chat_template_tool_calls::known_marker_candidates() { - match tool_call_format::try_parse(input, &candidate) { - ToolCallFormatOutcome::NoMatch => {} - ToolCallFormatOutcome::Parsed(calls) => { - let split = split_reasoning_prefix( - input, - reasoning_markers, - Some(&candidate.open), - is_partial, - ); - return Ok(ChatMessageParseOutcome::Recognized(ParsedChatMessage::new( - split.content, - split.reasoning, - calls, - ))); - } - ToolCallFormatOutcome::Failed(_shape_does_not_fit) => {} - } - } - - let via_ffi_result = self.parse_via_ffi(input, is_partial).map(|mut parsed| { - restore_partial_reasoning(&mut parsed, input, reasoning_markers, is_partial); - parsed - }); - - outcome_from_via_ffi_result(via_ffi_result, input, is_partial) - } - - fn parse_via_ffi( - &self, - input: &str, - is_partial: bool, - ) -> Result { - let input_cstring = - CString::new(input).map_err(ParseChatMessageError::InputContainsNulByte)?; - - let mut handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_parse_chat_message( - self.tools_parser.as_ptr(), - input_cstring.as_ptr(), - i32::from(is_partial), - &raw mut handle, - &raw mut out_error, - ) - }; - - let parsed = - unsafe { parse_chat_message_status_to_result(status, handle, &raw mut out_error) }; - - let mut free_error: *mut c_char = ptr::null_mut(); - let free_status = unsafe { - llama_cpp_bindings_sys::llama_rs_parsed_chat_free(handle, &raw mut free_error) - }; - let freed = unsafe { parsed_chat_free_status_to_result(free_status, free_error) }; - - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - - match parsed { - Ok(message) => freed.map(|()| message), - Err(parse_failure) => { - if let Err(destructor_failure) = freed { - log::error!("{destructor_failure}"); - } - - Err(parse_failure) - } - } - } -} - -impl Drop for ChatMessageParser { - fn drop(&mut self) { - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_chat_tools_parser_free( - self.tools_parser.as_ptr(), - &raw mut out_error, - ) - }; - - if let Err(destructor_failure) = - unsafe { chat_tools_parser_free_status_to_result(status, out_error) } - { - log::error!("{destructor_failure}"); - } - } -} - -#[cfg(test)] -mod tests { - use std::ffi::CStr; - use std::ffi::c_char; - use std::mem::discriminant; - use std::ptr; - - use llama_cpp_bindings_types::ModelProvidedToolCallId; - use llama_cpp_bindings_types::ParsedChatMessage; - use llama_cpp_bindings_types::ParsedToolCall; - use llama_cpp_bindings_types::ParsedToolCallId; - use llama_cpp_bindings_types::ReasoningMarkers; - use llama_cpp_bindings_types::ToolCallArguments; - use serde_json::json; - - use super::ReasoningSplit; - use super::chat_tools_parser_create_status_to_result; - use super::chat_tools_parser_free_status_to_result; - use super::outcome_from_via_ffi_result; - use super::parse_chat_message_status_to_result; - use super::parsed_chat_content_status_to_result; - use super::parsed_chat_free_status_to_result; - use super::parsed_chat_reasoning_content_status_to_result; - use super::parsed_chat_tool_call_arguments_status_to_result; - use super::parsed_chat_tool_call_count_status_to_result; - use super::parsed_chat_tool_call_id_status_to_result; - use super::parsed_chat_tool_call_name_status_to_result; - use super::restore_partial_reasoning; - use super::split_reasoning_prefix; - use super::tool_call_id_reported_by_llama_cpp; - use crate::chat_message_parse_outcome::ChatMessageParseOutcome; - use crate::error::parse_chat_message_error::ParseChatMessageError; - use crate::raw_chat_message::RawChatMessage; - #[test] - fn parse_chat_message_success_with_null_handle_is_contract_error() { - let mut out_error: *mut c_char = ptr::null_mut(); - let result = unsafe { - parse_chat_message_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_OK, - ptr::null_mut(), - &raw mut out_error, - ) - }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "success status contained a null parsed-chat handle", - } - )) - )); - } - - #[test] - fn parse_chat_message_allocation_failed_is_not_enough_memory() { - let mut out_error: *mut c_char = ptr::null_mut(); - let result = unsafe { - parse_chat_message_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - &raw mut out_error, - ) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::NotEnoughMemory) - ); - } - - #[test] - fn parse_chat_message_cxx_exception_is_message_unrecognized_and_nulls_error() { - let mut out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"the message could not be parsed".as_ptr()) - }; - let result = unsafe { - parse_chat_message_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null_mut(), - &raw mut out_error, - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::MessageUnrecognized { - message: "the message could not be parsed".to_owned() - }) - ); - - assert!( - out_error.is_null(), - "the reclaimed pointer must be nulled so the caller does not free it twice" - ); - } - - #[test] - fn parse_chat_message_cxx_exception_without_an_error_message_is_a_contract_error() { - let mut out_error: *mut c_char = ptr::null_mut(); - let result = unsafe { - parse_chat_message_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null_mut(), - &raw mut out_error, - ) - }; - - assert_eq!( - result.unwrap_err(), - ParseChatMessageError::FfiContract(crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "reported a thrown C++ exception without an error message", - }) - ); - } - - #[test] - fn parse_chat_message_unknown_status_is_preserved() { - let mut out_error: *mut c_char = ptr::null_mut(); - let result = unsafe { - parse_chat_message_status_to_result(255, ptr::null_mut(), &raw mut out_error) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_parse_chat_message", - code: 255, - })) - ); - } - - #[test] - fn parsed_chat_content_success_with_null_string_is_contract_error() { - let result = unsafe { - parsed_chat_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_OK, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_content", - detail: "success status contained a null string", - } - )) - )); - } - - #[test] - fn parsed_chat_content_allocation_failed_is_not_enough_memory() { - let result = unsafe { - parsed_chat_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::NotEnoughMemory) - ); - } - - #[test] - fn parsed_chat_content_cxx_exception_is_reported() { - let out_error = - unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"content read failed".as_ptr()) }; - let result = unsafe { - parsed_chat_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null_mut(), - out_error, - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::Reported { - message: "content read failed".to_owned() - }) - ); - } - - #[test] - fn parsed_chat_content_unknown_status_is_preserved() { - let result = - unsafe { parsed_chat_content_status_to_result(255, ptr::null_mut(), ptr::null_mut()) }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_content", - code: 255, - })) - )); - } - - #[test] - fn parsed_chat_reasoning_content_success_with_null_string_is_contract_error() { - let result = unsafe { - parsed_chat_reasoning_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_OK, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_reasoning_content", - detail: "success status contained a null string", - } - )) - )); - } - - #[test] - fn parsed_chat_reasoning_content_allocation_failed_is_not_enough_memory() { - let result = unsafe { - parsed_chat_reasoning_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::NotEnoughMemory) - ); - } - - #[test] - fn parsed_chat_reasoning_content_cxx_exception_is_reported() { - let out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"reasoning read failed".as_ptr()) - }; - let result = unsafe { - parsed_chat_reasoning_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null_mut(), - out_error, - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::Reported { - message: "reasoning read failed".to_owned() - }) - ); - } - - #[test] - fn parsed_chat_reasoning_content_unknown_status_is_preserved() { - let result = unsafe { - parsed_chat_reasoning_content_status_to_result(255, ptr::null_mut(), ptr::null_mut()) - }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_reasoning_content", - code: 255, - })) - )); - } - - #[test] - fn parsed_chat_tool_call_count_ok_returns_count() { - let result = unsafe { - parsed_chat_tool_call_count_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_OK, - 7, - ptr::null_mut(), - ) - }; - - assert_eq!(result.unwrap(), 7); - } - - #[test] - fn parsed_chat_tool_call_count_allocation_failed_is_not_enough_memory() { - let result = unsafe { - parsed_chat_tool_call_count_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_ERROR_STRING_ALLOCATION_FAILED, - 0, - ptr::null_mut(), - ) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::NotEnoughMemory) - ); - } - - #[test] - fn parsed_chat_tool_call_count_cxx_exception_is_reported() { - let out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"tool-call count failed".as_ptr()) - }; - let result = unsafe { - parsed_chat_tool_call_count_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_LLAMA_CPP_THREW_CXX_EXCEPTION, - 0, - out_error, - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::Reported { - message: "tool-call count failed".to_owned() - }) - ); - } - - #[test] - fn parsed_chat_tool_call_count_unknown_status_is_preserved() { - let result = - unsafe { parsed_chat_tool_call_count_status_to_result(255, 0, ptr::null_mut()) }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_tool_call_count", - code: 255, - })) - )); - } - - #[test] - fn parsed_chat_tool_call_id_success_with_null_string_is_contract_error() { - let result = unsafe { - parsed_chat_tool_call_id_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_OK, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_id", - detail: "success status contained a null string", - } - )) - )); - } - - #[test] - fn parsed_chat_tool_call_id_out_of_bounds_carries_index() { - let result = unsafe { - parsed_chat_tool_call_id_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_INDEX_OUT_OF_BOUNDS, - 4, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::ToolCallIdIndexOutOfBounds { index: 4 }) - ); - } - - #[test] - fn parsed_chat_tool_call_id_allocation_failed_is_not_enough_memory() { - let result = unsafe { - parsed_chat_tool_call_id_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_ERROR_STRING_ALLOCATION_FAILED, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::NotEnoughMemory) - ); - } - - #[test] - fn parsed_chat_tool_call_id_cxx_exception_is_reported() { - let out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"tool-call id read failed".as_ptr()) - }; - let result = unsafe { - parsed_chat_tool_call_id_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_LLAMA_CPP_THREW_CXX_EXCEPTION, - 0, - ptr::null_mut(), - out_error, - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::Reported { - message: "tool-call id read failed".to_owned() - }) - ); - } - - #[test] - fn parsed_chat_tool_call_id_unknown_status_is_preserved() { - let result = unsafe { - parsed_chat_tool_call_id_status_to_result(255, 0, ptr::null_mut(), ptr::null_mut()) - }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_tool_call_id", - code: 255, - })) - )); - } - - #[test] - fn parsed_chat_tool_call_name_success_with_null_string_is_contract_error() { - let result = unsafe { - parsed_chat_tool_call_name_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_OK, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_name", - detail: "success status contained a null string", - } - )) - )); - } - - #[test] - fn parsed_chat_tool_call_name_out_of_bounds_carries_index() { - let result = unsafe { - parsed_chat_tool_call_name_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_INDEX_OUT_OF_BOUNDS, - 2, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::ToolCallNameIndexOutOfBounds { index: 2 }) - ); - } - - #[test] - fn parsed_chat_tool_call_name_allocation_failed_is_not_enough_memory() { - let result = unsafe { - parsed_chat_tool_call_name_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_ERROR_STRING_ALLOCATION_FAILED, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::NotEnoughMemory) - ); - } - - #[test] - fn parsed_chat_tool_call_name_cxx_exception_is_reported() { - let out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"tool-call name read failed".as_ptr()) - }; - let result = unsafe { - parsed_chat_tool_call_name_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_LLAMA_CPP_THREW_CXX_EXCEPTION, - 0, - ptr::null_mut(), - out_error, - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::Reported { - message: "tool-call name read failed".to_owned() - }) - ); - } - - #[test] - fn parsed_chat_tool_call_name_unknown_status_is_preserved() { - let result = unsafe { - parsed_chat_tool_call_name_status_to_result(255, 0, ptr::null_mut(), ptr::null_mut()) - }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_tool_call_name", - code: 255, - })) - )); - } - - #[test] - fn parsed_chat_tool_call_arguments_success_with_null_string_is_contract_error() { - let result = unsafe { - parsed_chat_tool_call_arguments_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_OK, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_arguments", - detail: "success status contained a null string", - } - )) - )); - } - - #[test] - fn parsed_chat_tool_call_arguments_out_of_bounds_carries_index() { - let result = unsafe { - parsed_chat_tool_call_arguments_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_INDEX_OUT_OF_BOUNDS, - 9, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::ToolCallArgumentsIndexOutOfBounds { index: 9 }) - ); - } - - #[test] - fn parsed_chat_tool_call_arguments_allocation_failed_is_not_enough_memory() { - let result = unsafe { - parsed_chat_tool_call_arguments_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_ERROR_STRING_ALLOCATION_FAILED, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::NotEnoughMemory) - ); - } - - #[test] - fn parsed_chat_tool_call_arguments_cxx_exception_is_reported() { - let out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"tool-call arguments read failed".as_ptr()) - }; - let result = unsafe { - parsed_chat_tool_call_arguments_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_LLAMA_CPP_THREW_CXX_EXCEPTION, - 0, - ptr::null_mut(), - out_error, - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::Reported { - message: "tool-call arguments read failed".to_owned() - }) - ); - } - - #[test] - fn parsed_chat_tool_call_arguments_unknown_status_is_preserved() { - let result = unsafe { - parsed_chat_tool_call_arguments_status_to_result( - 255, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_tool_call_arguments", - code: 255, - })) - )); - } - - #[test] - fn split_reasoning_prefix_without_markers_returns_content_up_to_tool_call_open() { - let ReasoningSplit { reasoning, content } = - split_reasoning_prefix("answerrest", None, Some(""), false); - - assert_eq!(reasoning, ""); - assert_eq!(content, "answer"); - } - - #[test] - fn split_reasoning_prefix_with_missing_open_marker_returns_content_only() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let ReasoningSplit { reasoning, content } = - split_reasoning_prefix("plain answer", Some(&markers), Some(""), false); - - assert_eq!(reasoning, ""); - assert_eq!(content, "plain answer"); - } - - #[test] - fn split_reasoning_prefix_with_missing_close_marker_returns_content_only() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let ReasoningSplit { reasoning, content } = - split_reasoning_prefix("unterminated", Some(&markers), Some(""), false); - - assert_eq!(reasoning, ""); - assert_eq!(content, "unterminated"); - } - - #[test] - fn split_reasoning_prefix_with_partial_unclosed_marker_returns_reasoning() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let ReasoningSplit { reasoning, content } = split_reasoning_prefix( - "prefixunfinishedtail", - Some(&markers), - Some(""), - true, - ); - - assert_eq!(reasoning, "unfinished"); - assert_eq!(content, "prefix"); - } - - #[test] - fn split_reasoning_prefix_without_tool_marker_preserves_all_partial_reasoning() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let ReasoningSplit { reasoning, content } = - split_reasoning_prefix("unfinished", Some(&markers), None, true); - - assert_eq!(reasoning, "unfinished"); - assert_eq!(content, ""); - } - - #[test] - fn split_reasoning_prefix_extracts_reasoning_and_trailing_content() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let ReasoningSplit { reasoning, content } = split_reasoning_prefix( - "deduceanswertail", - Some(&markers), - Some(""), - false, - ); - - assert_eq!(reasoning, "deduce"); - assert_eq!(content, "answer"); - } - - #[test] - fn restore_partial_reasoning_preserves_non_partial_parser_result() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let mut parsed = - ParsedChatMessage::new("parsed content".to_owned(), String::new(), Vec::new()); - - restore_partial_reasoning(&mut parsed, "unfinished", Some(&markers), false); - - assert_eq!(parsed.content, "parsed content"); - assert_eq!(parsed.reasoning_content, ""); - } - - #[test] - fn restore_partial_reasoning_preserves_existing_reasoning() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let mut parsed = ParsedChatMessage::new( - "parsed content".to_owned(), - "parsed reasoning".to_owned(), - Vec::new(), - ); - - restore_partial_reasoning(&mut parsed, "plain response", Some(&markers), true); - - assert_eq!(parsed.content, "parsed content"); - assert_eq!(parsed.reasoning_content, "parsed reasoning"); - } - - #[test] - fn restore_partial_reasoning_removes_open_marker_from_parser_result() { - let markers = ReasoningMarkers { - open: "\n[THINK]\n".to_owned(), - closes: vec!["[/THINK]".to_owned()], - }; - let mut parsed = ParsedChatMessage::new( - String::new(), - "[THINK]parsed reasoning".to_owned(), - Vec::new(), - ); - - restore_partial_reasoning(&mut parsed, "complete response", Some(&markers), true); - - assert_eq!(parsed.content, ""); - assert_eq!(parsed.reasoning_content, "parsed reasoning"); - } - - #[test] - fn restore_partial_reasoning_preserves_unclosed_reasoning_whitespace() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let mut parsed = - ParsedChatMessage::new(String::new(), "normalized reasoning".to_owned(), Vec::new()); - - restore_partial_reasoning(&mut parsed, "\n\nreasoning", Some(&markers), true); - - assert_eq!(parsed.content, ""); - assert_eq!(parsed.reasoning_content, "\n\nreasoning"); - } - - #[test] - fn restore_partial_reasoning_preserves_closed_reasoning_whitespace() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let mut parsed = ParsedChatMessage::new( - "answer".to_owned(), - "normalized reasoning".to_owned(), - Vec::new(), - ); - - restore_partial_reasoning( - &mut parsed, - "\n\nreasoninganswer", - Some(&markers), - true, - ); - - assert_eq!(parsed.content, "answer"); - assert_eq!(parsed.reasoning_content, "\n\nreasoning"); - } - - #[test] - fn restore_partial_reasoning_removes_open_marker_after_parser_whitespace() { - let markers = ReasoningMarkers { - open: "\n[THINK]\n".to_owned(), - closes: vec!["[/THINK]".to_owned()], - }; - let mut parsed = ParsedChatMessage::new( - String::new(), - "\n[THINK]parsed reasoning".to_owned(), - Vec::new(), - ); - - restore_partial_reasoning(&mut parsed, "complete response", Some(&markers), true); - - assert_eq!(parsed.content, ""); - assert_eq!(parsed.reasoning_content, "parsed reasoning"); - } - - #[test] - fn restore_partial_reasoning_preserves_result_without_open_marker() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let mut parsed = - ParsedChatMessage::new("parsed content".to_owned(), String::new(), Vec::new()); - - restore_partial_reasoning(&mut parsed, "unfinished", Some(&markers), true); - - assert_eq!(parsed.content, "parsed content"); - assert_eq!(parsed.reasoning_content, ""); - } - - #[test] - fn restore_partial_reasoning_recovers_unclosed_reasoning() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let mut parsed = - ParsedChatMessage::new("unfinished".to_owned(), String::new(), Vec::new()); - - restore_partial_reasoning(&mut parsed, "unfinished", Some(&markers), true); - - assert_eq!(parsed.content, ""); - assert_eq!(parsed.reasoning_content, "unfinished"); - } - - #[test] - fn outcome_from_via_ffi_result_recognized_keeps_a_tool_call_the_model_gave_no_id() { - let parsed = ParsedChatMessage::new( - "answer".to_owned(), - String::new(), - vec![ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - "tool".to_owned(), - ToolCallArguments::ValidJson(json!({})), - )], - ); - - let outcome = outcome_from_via_ffi_result(Ok(parsed.clone()), "answer", false); - - assert_eq!( - outcome.unwrap(), - ChatMessageParseOutcome::Recognized(parsed) - ); - } - - #[test] - fn an_empty_llama_cpp_tool_call_id_is_not_provided_by_the_model() { - assert_eq!( - tool_call_id_reported_by_llama_cpp(String::new()), - ParsedToolCallId::NotProvidedByModel - ); - } - - #[test] - fn a_non_empty_llama_cpp_tool_call_id_is_provided_by_the_model() { - assert_eq!( - Ok(tool_call_id_reported_by_llama_cpp("abc123def".to_owned())), - ModelProvidedToolCallId::try_from("abc123def".to_owned()) - .map(ParsedToolCallId::ModelProvided) - ); - } - - #[test] - fn outcome_from_via_ffi_result_message_unrecognized_is_unrecognized_with_raw_message() { - let outcome = outcome_from_via_ffi_result( - Err(ParseChatMessageError::MessageUnrecognized { - message: "boom".to_owned(), - }), - "garbled", - true, - ); - - assert_eq!( - outcome.unwrap(), - ChatMessageParseOutcome::Unrecognized(RawChatMessage { - text: "garbled".to_owned(), - is_partial: true, - ffi_error_message: "boom".to_owned(), - }) - ); - } - - #[test] - fn outcome_from_via_ffi_result_parser_creation_failure_propagates() { - let outcome = outcome_from_via_ffi_result( - Err(ParseChatMessageError::ParserCreationFailed { - message: "the parser could not be built".to_owned(), - }), - "garbled", - true, - ); - - assert_eq!( - discriminant(&outcome.unwrap_err()), - discriminant(&ParseChatMessageError::ParserCreationFailed { - message: String::new() - }) - ); - } - - #[test] - fn outcome_from_via_ffi_result_other_error_propagates() { - let outcome = outcome_from_via_ffi_result(Err(ParseChatMessageError::NoVocab), "x", false); - - assert_eq!( - discriminant(&outcome.unwrap_err()), - discriminant(&ParseChatMessageError::NoVocab) - ); - } - - #[test] - fn parsed_chat_free_ok_is_success() { - let result = unsafe { - parsed_chat_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_OK, - ptr::null_mut(), - ) - }; - - assert!( - result.is_ok(), - "a clean destructor must not report a failure" - ); - } - - #[test] - fn parsed_chat_free_allocation_failed_is_not_enough_memory() { - let result = unsafe { - parsed_chat_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - ) - }; - - assert_eq!(result.err(), Some(ParseChatMessageError::NotEnoughMemory)); - } - - #[test] - fn parsed_chat_free_llama_cpp_out_of_memory_is_preserved() { - let result = unsafe { - parsed_chat_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null_mut(), - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::LlamaCppOutOfMemory) - ); - } - - #[test] - fn parsed_chat_free_destructor_threw_surfaces_the_message() { - let out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"the destructor threw".as_ptr()) - }; - let result = unsafe { - parsed_chat_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - out_error, - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::DestructorFailed { - message: "the destructor threw".to_owned() - }) - ); - } - - #[test] - fn parsed_chat_free_unknown_status_is_preserved() { - let result = unsafe { parsed_chat_free_status_to_result(255, ptr::null_mut()) }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_free", - code: 255, - })) - ); - } - - #[test] - fn parse_chat_message_status_to_result_maps_null_tools_parser_arg() { - let mut out_error_slot: *mut c_char = ptr::null_mut(); - let outcome = unsafe { - parse_chat_message_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_TOOLS_PARSER_ARG, - ptr::null_mut(), - &raw mut out_error_slot, - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "was given a null tools parser argument", - } - )) - ); - } - - #[test] - fn parse_chat_message_status_to_result_maps_null_input_arg() { - let mut out_error_slot: *mut c_char = ptr::null_mut(); - let outcome = unsafe { - parse_chat_message_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_INPUT_ARG, - ptr::null_mut(), - &raw mut out_error_slot, - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "was given a null input argument", - } - )) - ); - } - - #[test] - fn parse_chat_message_status_to_result_maps_null_out_handle_arg() { - let mut out_error_slot: *mut c_char = ptr::null_mut(); - let outcome = unsafe { - parse_chat_message_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_OUT_HANDLE_ARG, - ptr::null_mut(), - &raw mut out_error_slot, - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "was given a null out_handle argument", - } - )) - ); - } - - #[test] - fn parse_chat_message_status_to_result_maps_null_out_error_arg() { - let mut out_error_slot: *mut c_char = ptr::null_mut(); - let outcome = unsafe { - parse_chat_message_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_OUT_ERROR_ARG, - ptr::null_mut(), - &raw mut out_error_slot, - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "was given a null out_error argument", - } - )) - ); - } - - #[test] - fn parse_chat_message_status_to_result_maps_llama_cpp_out_of_memory() { - let mut out_error_slot: *mut c_char = ptr::null_mut(); - let outcome = unsafe { - parse_chat_message_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null_mut(), - &raw mut out_error_slot, - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::LlamaCppOutOfMemory) - ); - } - - #[test] - fn parsed_chat_content_status_to_result_maps_null_handle_arg() { - let outcome = unsafe { - parsed_chat_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_NULL_HANDLE_ARG, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_content", - detail: "was given a null handle argument", - } - )) - ); - } - - #[test] - fn parsed_chat_content_status_to_result_maps_null_out_string_arg() { - let outcome = unsafe { - parsed_chat_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_NULL_OUT_STRING_ARG, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_content", - detail: "was given a null out_string argument", - } - )) - ); - } - - #[test] - fn parsed_chat_content_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = unsafe { - parsed_chat_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::LlamaCppOutOfMemory) - ); - } - - #[test] - fn parsed_chat_reasoning_content_status_to_result_maps_null_handle_arg() { - let outcome = unsafe { - parsed_chat_reasoning_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_NULL_HANDLE_ARG, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_reasoning_content", - detail: "was given a null handle argument", - } - )) - ); - } - - #[test] - fn parsed_chat_reasoning_content_status_to_result_maps_null_out_string_arg() { - let outcome = unsafe { - parsed_chat_reasoning_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_NULL_OUT_STRING_ARG, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_reasoning_content", - detail: "was given a null out_string argument", - } - )) - ); - } - - #[test] - fn parsed_chat_reasoning_content_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = unsafe { - parsed_chat_reasoning_content_status_to_result(llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), ptr::null_mut()) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::LlamaCppOutOfMemory) - ); - } - - #[test] - fn parsed_chat_tool_call_count_status_to_result_maps_null_handle_arg() { - let outcome = unsafe { - parsed_chat_tool_call_count_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_NULL_HANDLE_ARG, - 0, - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_count", - detail: "was given a null handle argument", - } - )) - ); - } - - #[test] - fn parsed_chat_tool_call_count_status_to_result_maps_null_out_count_arg() { - let outcome = unsafe { - parsed_chat_tool_call_count_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_NULL_OUT_COUNT_ARG, - 0, - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_count", - detail: "was given a null out_count argument", - } - )) - ); - } - - #[test] - fn parsed_chat_tool_call_count_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = unsafe { - parsed_chat_tool_call_count_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_LLAMA_CPP_OUT_OF_MEMORY, - 0, - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::LlamaCppOutOfMemory) - ); - } - - #[test] - fn parsed_chat_tool_call_id_status_to_result_maps_null_handle_arg() { - let outcome = unsafe { - parsed_chat_tool_call_id_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_NULL_HANDLE_ARG, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_id", - detail: "was given a null handle argument", - } - )) - ); - } - - #[test] - fn parsed_chat_tool_call_id_status_to_result_maps_null_out_string_arg() { - let outcome = unsafe { - parsed_chat_tool_call_id_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_NULL_OUT_STRING_ARG, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_id", - detail: "was given a null out_string argument", - } - )) - ); - } - - #[test] - fn parsed_chat_tool_call_id_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = unsafe { - parsed_chat_tool_call_id_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_LLAMA_CPP_OUT_OF_MEMORY, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::LlamaCppOutOfMemory) - ); - } - - #[test] - fn parsed_chat_tool_call_name_status_to_result_maps_null_handle_arg() { - let outcome = unsafe { - parsed_chat_tool_call_name_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_NULL_HANDLE_ARG, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_name", - detail: "was given a null handle argument", - } - )) - ); - } - - #[test] - fn parsed_chat_tool_call_name_status_to_result_maps_null_out_string_arg() { - let outcome = unsafe { - parsed_chat_tool_call_name_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_NULL_OUT_STRING_ARG, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_name", - detail: "was given a null out_string argument", - } - )) - ); - } - - #[test] - fn parsed_chat_tool_call_name_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = unsafe { - parsed_chat_tool_call_name_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_LLAMA_CPP_OUT_OF_MEMORY, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::LlamaCppOutOfMemory) - ); - } - - #[test] - fn parsed_chat_tool_call_arguments_status_to_result_maps_null_handle_arg() { - let outcome = unsafe { - parsed_chat_tool_call_arguments_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NULL_HANDLE_ARG, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_arguments", - detail: "was given a null handle argument", - } - )) - ); - } - - #[test] - fn parsed_chat_tool_call_arguments_status_to_result_maps_null_out_string_arg() { - let outcome = unsafe { - parsed_chat_tool_call_arguments_status_to_result(llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NULL_OUT_STRING_ARG, 0, ptr::null_mut(), ptr::null_mut()) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_arguments", - detail: "was given a null out_string argument", - } - )) - ); - } - - #[test] - fn parsed_chat_tool_call_arguments_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = unsafe { - parsed_chat_tool_call_arguments_status_to_result(llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_LLAMA_CPP_OUT_OF_MEMORY, 0, ptr::null_mut(), ptr::null_mut()) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::LlamaCppOutOfMemory) - ); - } - - fn tools_parser_creation_error( - status: llama_cpp_bindings_sys::llama_rs_chat_tools_parser_create_status, - message: Option<&CStr>, - ) -> ParseChatMessageError { - let mut out_error = message.map_or(ptr::null_mut(), |message| unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(message.as_ptr()) - }); - - let error = unsafe { - chat_tools_parser_create_status_to_result(status, ptr::null_mut(), &raw mut out_error) - } - .expect_err("the status must map to an error"); - - assert!(out_error.is_null(), "a reported message must be reclaimed"); - - error - } - - #[test] - fn tools_parser_creation_reports_a_success_without_a_parser_as_a_contract_error() { - assert_eq!( - tools_parser_creation_error( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_OK, - None - ), - ParseChatMessageError::FfiContract(crate::FfiContractError { - operation: "llama_rs_chat_tools_parser_create", - detail: "success status contained a null tools parser handle", - }) - ); - } - - #[test] - fn tools_parser_creation_reports_tools_that_are_not_an_array() { - assert_eq!( - tools_parser_creation_error( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_TOOLS_NOT_AN_ARRAY, - None - ), - ParseChatMessageError::ToolsNotAnArray - ); - } - - #[test] - fn tools_parser_creation_reports_running_out_of_memory() { - assert_eq!( - tools_parser_creation_error( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_ERROR_STRING_ALLOCATION_FAILED, - None - ), - ParseChatMessageError::NotEnoughMemory - ); - assert_eq!( - tools_parser_creation_error( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_OUT_OF_MEMORY, - None - ), - ParseChatMessageError::LlamaCppOutOfMemory - ); - } - - #[test] - fn tools_parser_creation_reports_the_build_failure_message() { - assert_eq!( - tools_parser_creation_error( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, - Some(c"tools do not fit the template") - ), - ParseChatMessageError::ToolsParserBuildFailed { - message: "tools do not fit the template".to_owned() - } - ); - } - - #[test] - fn tools_parser_creation_reports_a_build_failure_without_a_message_as_a_contract_error() { - assert_eq!( - tools_parser_creation_error( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, - None - ), - ParseChatMessageError::FfiContract(crate::FfiContractError { - operation: "llama_rs_chat_tools_parser_create", - detail: "reported a thrown C++ exception without an error message", - }) - ); - } - - #[test] - fn tools_parser_creation_reports_every_null_argument_as_a_contract_error() { - for (status, detail) in [ - ( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_PARSER_ARG, - "was given a null parser argument", - ), - ( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_TOOLS_JSON_ARG, - "was given a null tools_json argument", - ), - ( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_OUT_TOOLS_PARSER_ARG, - "was given a null out_tools_parser argument", - ), - ( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_OUT_ERROR_ARG, - "was given a null out_error argument", - ), - ] { - assert_eq!( - tools_parser_creation_error(status, None), - ParseChatMessageError::FfiContract(crate::FfiContractError { - operation: "llama_rs_chat_tools_parser_create", - detail, - }) - ); - } - } - - #[test] - fn tools_parser_creation_preserves_an_unknown_status() { - assert_eq!( - tools_parser_creation_error(255, None), - ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_chat_tools_parser_create", - code: 255, - }) - ); - } - - #[test] - fn tools_parser_release_maps_every_status() { - assert_eq!( - unsafe { - chat_tools_parser_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_FREE_OK, - ptr::null_mut(), - ) - }, - Ok(()) - ); - assert_eq!( - unsafe { - chat_tools_parser_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_FREE_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - ) - }, - Err(ParseChatMessageError::NotEnoughMemory) - ); - assert_eq!( - unsafe { - chat_tools_parser_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_FREE_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null_mut(), - ) - }, - Err(ParseChatMessageError::LlamaCppOutOfMemory) - ); - assert_eq!( - unsafe { - chat_tools_parser_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - llama_cpp_bindings_sys::llama_rs_string_dup(c"destructor failed".as_ptr()), - ) - }, - Err(ParseChatMessageError::DestructorFailed { - message: "destructor failed".to_owned() - }) - ); - assert_eq!( - unsafe { chat_tools_parser_free_status_to_result(255, ptr::null_mut()) }, - Err(ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_chat_tools_parser_free", - code: 255, - })) - ); - } -} diff --git a/llama-cpp-bindings/src/chat_output.rs b/llama-cpp-bindings/src/chat_output.rs new file mode 100644 index 000000000..9a85cabc0 --- /dev/null +++ b/llama-cpp-bindings/src/chat_output.rs @@ -0,0 +1,108 @@ +use llama_cpp_bindings_sys::LLAMA_TOKEN_NULL; +use llama_cpp_bindings_sys::llama_token; + +use crate::chat_output_token::ChatOutputToken; +use crate::token::LlamaToken; + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct ChatOutput { + text: String, + byte_tokens: Vec, + tokens: Vec, +} + +impl From<&str> for ChatOutput { + /// Holds `text` without tokens, the way `common_chat_input` holds plain text. + fn from(text: &str) -> Self { + Self { + text: text.to_owned(), + byte_tokens: vec![LLAMA_TOKEN_NULL; text.len()], + tokens: Vec::new(), + } + } +} + +impl ChatOutput { + /// Appends the text `token` decoded to, recording the token at the first byte of its piece + /// the way `common_chat_input` lays tokens over the text. + pub fn push(&mut self, token: LlamaToken, piece: &str) { + let start = self.text.len(); + + self.text.push_str(piece); + self.byte_tokens.extend((0..piece.len()).map(|offset| { + if offset == 0 { + token.0 + } else { + LLAMA_TOKEN_NULL + } + })); + self.tokens.push(ChatOutputToken { + token, + bytes: start..self.text.len(), + }); + } + + #[must_use] + pub fn text(&self) -> &str { + &self.text + } + + #[must_use] + pub fn byte_tokens(&self) -> &[llama_token] { + &self.byte_tokens + } + + #[must_use] + pub fn tokens(&self) -> &[ChatOutputToken] { + &self.tokens + } +} + +#[cfg(test)] +mod tests { + use llama_cpp_bindings_sys::LLAMA_TOKEN_NULL; + + use super::ChatOutput; + use crate::chat_output_token::ChatOutputToken; + use crate::token::LlamaToken; + + #[test] + fn each_token_owns_the_bytes_of_its_piece_and_marks_its_first_byte() { + let mut output = ChatOutput::default(); + + output.push(LlamaToken::new(7), ""); + output.push(LlamaToken::new(9), ""); + output.push(LlamaToken::new(11), "hi"); + + assert_eq!(output.text(), "hi"); + assert_eq!( + output.byte_tokens(), + [7, LLAMA_TOKEN_NULL, LLAMA_TOKEN_NULL, 11, LLAMA_TOKEN_NULL] + ); + assert_eq!( + output.tokens(), + [ + ChatOutputToken { + token: LlamaToken::new(7), + bytes: 0..3, + }, + ChatOutputToken { + token: LlamaToken::new(9), + bytes: 3..3, + }, + ChatOutputToken { + token: LlamaToken::new(11), + bytes: 3..5, + }, + ] + ); + } + + #[test] + fn plain_text_carries_no_tokens() { + let output = ChatOutput::from("ab"); + + assert_eq!(output.byte_tokens(), [LLAMA_TOKEN_NULL, LLAMA_TOKEN_NULL]); + assert_eq!(output.tokens(), []); + } +} diff --git a/llama-cpp-bindings/src/chat_output_completeness.rs b/llama-cpp-bindings/src/chat_output_completeness.rs new file mode 100644 index 000000000..38d7d3024 --- /dev/null +++ b/llama-cpp-bindings/src/chat_output_completeness.rs @@ -0,0 +1,5 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ChatOutputCompleteness { + Partial, + Finished, +} diff --git a/llama-cpp-bindings/src/chat_output_token.rs b/llama-cpp-bindings/src/chat_output_token.rs new file mode 100644 index 000000000..6e30f52d4 --- /dev/null +++ b/llama-cpp-bindings/src/chat_output_token.rs @@ -0,0 +1,9 @@ +use std::ops::Range; + +use crate::token::LlamaToken; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ChatOutputToken { + pub token: LlamaToken, + pub bytes: Range, +} diff --git a/llama-cpp-bindings/src/chat_piece_role.rs b/llama-cpp-bindings/src/chat_piece_role.rs new file mode 100644 index 000000000..6c69c497e --- /dev/null +++ b/llama-cpp-bindings/src/chat_piece_role.rs @@ -0,0 +1,16 @@ +use crate::sampled_token_section::SampledTokenSection; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ChatPieceRole { + Text(SampledTokenSection), + Marker(SampledTokenSection), +} + +impl ChatPieceRole { + #[must_use] + pub const fn section(self) -> SampledTokenSection { + match self { + Self::Text(section) | Self::Marker(section) => section, + } + } +} diff --git a/llama-cpp-bindings/src/chat_prompt_part.rs b/llama-cpp-bindings/src/chat_prompt_part.rs new file mode 100644 index 000000000..1b8d5f7d7 --- /dev/null +++ b/llama-cpp-bindings/src/chat_prompt_part.rs @@ -0,0 +1,14 @@ +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ChatPromptPart { + Template(String), + Input(String), +} + +impl ChatPromptPart { + #[must_use] + pub fn text(&self) -> &str { + match self { + Self::Template(text) | Self::Input(text) => text, + } + } +} diff --git a/llama-cpp-bindings/src/chat_session.rs b/llama-cpp-bindings/src/chat_session.rs new file mode 100644 index 000000000..cee8ed08e --- /dev/null +++ b/llama-cpp-bindings/src/chat_session.rs @@ -0,0 +1,1538 @@ +use std::ffi::CString; +use std::ffi::c_char; +use std::ptr; +use std::ptr::NonNull; + +use llama_cpp_bindings_types::ParsedChatMessage; +use llama_cpp_ffi_status::read_and_free_cpp_string; +use serde_json::Value; +use serde_json::json; + +use crate::chat_assistant_prefill::ChatAssistantPrefill; +use crate::chat_assistant_turn::ChatAssistantTurn; +use crate::chat_classification::ChatClassification; +use crate::chat_grammar_trigger::ChatGrammarTrigger; +use crate::chat_message_parse_outcome::ChatMessageParseOutcome; +use crate::chat_output::ChatOutput; +use crate::chat_output_completeness::ChatOutputCompleteness; +use crate::chat_prompt_part::ChatPromptPart; +use crate::chat_session_options::ChatSessionOptions; +use crate::chat_span::ChatSpan; +use crate::chat_span_kind::ChatSpanKind; +use crate::chat_spans::ChatSpans; +use crate::chat_template_tool_calls::known_marker_candidates; +use crate::cpp_text_view::cpp_text_view; +use crate::error::chat_session_error::ChatSessionError; +use crate::error::grammar_error::GrammarError; +use crate::error::parse_chat_message_error::ParseChatMessageError; +use crate::model::AddBos; +use crate::model::LlamaModel; +use crate::model::ParseSpecialTokens; +use crate::model::llama_chat_message::LlamaChatMessage; +use crate::parsed_chat::take_parsed_chat_message; +use crate::raw_chat_message::RawChatMessage; +use crate::result_after_release::result_after_release; +use crate::sampling::LlamaSampler; +use crate::token::LlamaToken; +use crate::tool_call_format::ToolCallFormatOutcome; +use crate::tool_call_format::try_parse; + +const GRAMMAR_ROOT: &str = "root"; + +fn chat_message_json(message: &LlamaChatMessage) -> Result { + Ok(json!({ + "role": message.role.to_str().map_err(ChatSessionError::MessageNotUtf8)?, + "content": message.content.to_str().map_err(ChatSessionError::MessageNotUtf8)?, + })) +} + +fn continued_assistant_message_json(assistant_turn: &ChatAssistantTurn) -> Option { + match assistant_turn { + ChatAssistantTurn::Omitted | ChatAssistantTurn::New => None, + ChatAssistantTurn::ContinueReasoning(reasoning_content) => Some(json!({ + "role": "assistant", + "content": "", + "reasoning_content": reasoning_content, + })), + ChatAssistantTurn::ContinueContent(ChatAssistantPrefill { + reasoning_content, + content, + }) => Some(json!({ + "role": "assistant", + "content": content, + "reasoning_content": reasoning_content, + })), + } +} + +fn messages_json( + messages: &[LlamaChatMessage], + assistant_turn: &ChatAssistantTurn, +) -> Result { + let mut messages = messages + .iter() + .map(chat_message_json) + .collect::, ChatSessionError>>()?; + + messages.extend(continued_assistant_message_json(assistant_turn)); + + CString::new(Value::Array(messages).to_string()).map_err(ChatSessionError::RequestContainsNul) +} + +fn session_text(text: *const c_char, length: usize) -> Result { + unsafe { cpp_text_view(text, length) } + .map(str::to_owned) + .map_err(ChatSessionError::SessionTextNotUtf8) +} + +fn anchored_pattern(pattern: &str) -> String { + if pattern.is_empty() { + return "^$".to_owned(); + } + + let start = if pattern.starts_with('^') { "" } else { "^" }; + let end = if pattern.ends_with('$') { "" } else { "$" }; + + format!("{start}{pattern}{end}") +} + +const fn contract_violation( + operation: &'static str, + detail: &'static str, +) -> crate::FfiContractError { + crate::FfiContractError { operation, detail } +} + +fn chat_session_create_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_chat_session_create_status, + out_session: llama_cpp_bindings_sys::llama_rs_chat_session_handle, + out_error: *mut c_char, +) -> Result, ChatSessionError> { + const OPERATION: &str = "llama_rs_chat_session_create"; + let reported_message = || { + unsafe { + read_and_free_cpp_string( + out_error, + OPERATION, + "reported a failure without an error message", + ) + } + .map_err(ChatSessionError::from) + }; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_OK => NonNull::new(out_session) + .ok_or_else(|| { + contract_violation(OPERATION, "success status contained a null session").into() + }), + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_MESSAGES_NOT_JSON => { + Err(ChatSessionError::MessagesNotJson { + message: reported_message()?, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_MESSAGES_INVALID => { + Err(ChatSessionError::MessagesInvalid { + message: reported_message()?, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_NOT_JSON => { + Err(ChatSessionError::ToolsNotJson { + message: reported_message()?, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_INVALID => { + Err(ChatSessionError::ToolsInvalid { + message: reported_message()?, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TEMPLATE_CANNOT_SERVE_REQUEST => { + Err(ChatSessionError::TemplateCannotServeRequest { + message: reported_message()?, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION => { + Err(ChatSessionError::Reported { + message: reported_message()?, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED => { + Err(ChatSessionError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_LLAMA_CPP_OUT_OF_MEMORY => { + Err(ChatSessionError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_TEMPLATES_ARG => { + Err(contract_violation(OPERATION, "was given a null templates argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_VOCAB_ARG => { + Err(contract_violation(OPERATION, "was given a null vocab argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_MESSAGES_JSON_ARG => { + Err(contract_violation(OPERATION, "was given a null messages_json argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_TOOLS_JSON_ARG => { + Err(contract_violation(OPERATION, "was given a null tools_json argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_SESSION_ARG => { + Err(contract_violation(OPERATION, "was given a null out_session argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_ERROR_ARG => { + Err(contract_violation(OPERATION, "was given a null out_error argument").into()) + } + other => Err(crate::FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), + } +} + +fn chat_session_free_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_chat_session_free_status, + out_error: *mut c_char, +) -> Result<(), ChatSessionError> { + const OPERATION: &str = "llama_rs_chat_session_free"; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_FREE_OK => Ok(()), + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_FREE_ERROR_STRING_ALLOCATION_FAILED => { + Err(ChatSessionError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_FREE_LLAMA_CPP_OUT_OF_MEMORY => { + Err(ChatSessionError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + OPERATION, + "reported a thrown C++ exception without an error message", + ) + }?; + Err(ChatSessionError::Reported { message }) + } + other => Err(crate::FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), + } +} + +fn chat_session_parse_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_chat_session_parse_status, + out_handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, + out_error: *mut c_char, + output: &ChatOutput, + completeness: ChatOutputCompleteness, +) -> Result { + const OPERATION: &str = "llama_rs_chat_session_parse"; + let reported_message = || unsafe { + read_and_free_cpp_string( + out_error, + OPERATION, + "reported a failure without an error message", + ) + }; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_OK => NonNull::new(out_handle) + .ok_or_else(|| { + contract_violation(OPERATION, "success status contained a null handle").into() + }) + .and_then(take_parsed_chat_message) + .map(ChatMessageParseOutcome::Recognized), + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_OUTPUT_DOES_NOT_MATCH_FORMAT => { + Ok(ChatMessageParseOutcome::Unrecognized(RawChatMessage { + text: output.text().to_owned(), + completeness, + ffi_error_message: reported_message()?, + })) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION => { + Err(ParseChatMessageError::Reported { + message: reported_message()?, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_ERROR_STRING_ALLOCATION_FAILED => { + Err(ParseChatMessageError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_OUT_OF_MEMORY => { + Err(ParseChatMessageError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_SESSION_ARG => { + Err(contract_violation(OPERATION, "was given a null session argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_TEXT_ARG => { + Err(contract_violation(OPERATION, "was given a null text argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_BYTE_TOKENS_ARG => { + Err(contract_violation(OPERATION, "was given a null byte_tokens argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_HANDLE_ARG => { + Err(contract_violation(OPERATION, "was given a null out_handle argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_ERROR_ARG => { + Err(contract_violation(OPERATION, "was given a null out_error argument").into()) + } + other => Err(crate::FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), + } +} + +fn chat_spans_free_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_chat_spans_free_status, + free_error: *mut c_char, +) -> Result<(), ParseChatMessageError> { + const OPERATION: &str = "llama_rs_chat_spans_free"; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPANS_FREE_OK => Ok(()), + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPANS_FREE_ERROR_STRING_ALLOCATION_FAILED => { + Err(ParseChatMessageError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPANS_FREE_LLAMA_CPP_OUT_OF_MEMORY => { + Err(ParseChatMessageError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPANS_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION => { + Err(ParseChatMessageError::DestructorFailed { + message: unsafe { + read_and_free_cpp_string( + free_error, + OPERATION, + "reported a thrown C++ exception without an error message", + ) + }?, + }) + } + other => Err(crate::FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), + } +} + +fn read_chat_spans( + spans: NonNull, + completeness: ChatOutputCompleteness, +) -> Result { + let count = unsafe { llama_cpp_bindings_sys::llama_rs_chat_spans_count(spans.as_ptr()) }; + let read = (0..count) + .map(|index| { + let kind = ChatSpanKind::try_from(unsafe { + llama_cpp_bindings_sys::llama_rs_chat_spans_kind(spans.as_ptr(), index) + })?; + let start = + unsafe { llama_cpp_bindings_sys::llama_rs_chat_spans_start(spans.as_ptr(), index) }; + let end = + unsafe { llama_cpp_bindings_sys::llama_rs_chat_spans_end(spans.as_ptr(), index) }; + + Ok(ChatSpan { + kind, + bytes: start..end, + }) + }) + .collect::, ParseChatMessageError>>() + .map(|spans_read| ChatSpans { + parsed_length: unsafe { + llama_cpp_bindings_sys::llama_rs_chat_spans_parsed_length(spans.as_ptr()) + }, + spans: spans_read, + completeness, + }); + let mut free_error: *mut c_char = ptr::null_mut(); + let free_status = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_spans_free(spans.as_ptr(), &raw mut free_error) + }; + + result_after_release( + read, + chat_spans_free_status_to_result(free_status, free_error), + ) +} + +fn chat_session_classify_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_chat_session_classify_status, + out_spans: *mut llama_cpp_bindings_sys::llama_rs_chat_spans, + out_error: *mut c_char, + completeness: ChatOutputCompleteness, +) -> Result { + const OPERATION: &str = "llama_rs_chat_session_classify"; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_OK => NonNull::new(out_spans) + .ok_or_else(|| { + contract_violation(OPERATION, "success status contained null spans").into() + }) + .and_then(|spans| read_chat_spans(spans, completeness)) + .map(ChatClassification::Spans), + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_OUTPUT_DOES_NOT_MATCH_FORMAT => { + Ok(ChatClassification::OutputDoesNotMatchFormat) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_THREW_CXX_EXCEPTION => { + Err(ParseChatMessageError::Reported { + message: unsafe { + read_and_free_cpp_string( + out_error, + OPERATION, + "reported a thrown C++ exception without an error message", + ) + }?, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_ERROR_STRING_ALLOCATION_FAILED => { + Err(ParseChatMessageError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_OUT_OF_MEMORY => { + Err(ParseChatMessageError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_SESSION_ARG => { + Err(contract_violation(OPERATION, "was given a null session argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_TEXT_ARG => { + Err(contract_violation(OPERATION, "was given a null text argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_BYTE_TOKENS_ARG => { + Err(contract_violation(OPERATION, "was given a null byte_tokens argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_SPANS_ARG => { + Err(contract_violation(OPERATION, "was given a null out_spans argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_ERROR_ARG => { + Err(contract_violation(OPERATION, "was given a null out_error argument").into()) + } + other => Err(crate::FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), + } +} + +fn regex_escape_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_regex_escape_status, + out_string: *mut c_char, + out_error: *mut c_char, +) -> Result { + const OPERATION: &str = "llama_rs_regex_escape"; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_OK => Ok(unsafe { + read_and_free_cpp_string( + out_string, + OPERATION, + "success status contained a null string", + ) + }?), + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_THREW_CXX_EXCEPTION => { + Err(GrammarError::Reported { + message: unsafe { + read_and_free_cpp_string( + out_error, + OPERATION, + "reported a thrown C++ exception without an error message", + ) + }?, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_ERROR_STRING_ALLOCATION_FAILED => { + Err(GrammarError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_OUT_OF_MEMORY => { + Err(GrammarError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_TEXT_ARG => { + Err(contract_violation(OPERATION, "was given a null text argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_OUT_STRING_ARG => { + Err(contract_violation(OPERATION, "was given a null out_string argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_OUT_ERROR_ARG => { + Err(contract_violation(OPERATION, "was given a null out_error argument").into()) + } + other => Err(crate::FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), + } +} + +fn regex_escaped(word: &str) -> Result { + let mut out_string: *mut c_char = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_regex_escape( + word.as_ptr().cast::(), + word.len(), + &raw mut out_string, + &raw mut out_error, + ) + }; + + regex_escape_status_to_result(status, out_string, out_error) +} + +/// One chat request rendered by the model's chat template the way llama-server renders it: the +/// prompt, the grammar that constrains tool calls, and the parser of the model's answer. +pub struct ChatSession<'model> { + model: &'model LlamaModel, + session: NonNull, + supports_tool_calls: bool, +} + +unsafe impl Send for ChatSession<'_> {} + +impl<'model> ChatSession<'model> { + /// Renders `messages` with `tools_json`, an `OpenAI` tools array, through the model's chat + /// template. + /// + /// # Errors + /// + /// Returns the [`ChatSessionError`] describing why the model's chat template cannot serve the + /// request. + pub fn new( + model: &'model LlamaModel, + messages: &[LlamaChatMessage], + tools_json: &str, + options: &ChatSessionOptions, + ) -> Result { + let templates = model.chat_templates()?; + let messages_json = messages_json(messages, &options.assistant_turn)?; + let tools_json = CString::new(tools_json).map_err(ChatSessionError::RequestContainsNul)?; + let mut out_session: llama_cpp_bindings_sys::llama_rs_chat_session_handle = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_create( + templates.as_ptr(), + model.vocab_ptr(), + messages_json.as_ptr(), + tools_json.as_ptr(), + options.into(), + &raw mut out_session, + &raw mut out_error, + ) + }; + + chat_session_create_status_to_result(status, out_session, out_error).map(|session| Self { + model, + session, + supports_tool_calls: templates.supports_tool_calls(), + }) + } + + fn text_of( + &self, + read: impl FnOnce( + *mut llama_cpp_bindings_sys::llama_rs_chat_session, + *mut usize, + ) -> *const c_char, + ) -> Result { + let mut length = 0; + let text = read(self.session.as_ptr(), &raw mut length); + + session_text(text, length) + } + + fn texts_of( + &self, + count: usize, + read: impl Fn( + *mut llama_cpp_bindings_sys::llama_rs_chat_session, + usize, + *mut usize, + ) -> *const c_char, + ) -> Result, ChatSessionError> { + (0..count) + .map(|index| self.text_of(|session, out_length| read(session, index, out_length))) + .collect() + } + + /// # Errors + /// + /// Returns [`ChatSessionError::SessionTextNotUtf8`] when the rendered prompt is not UTF-8. + pub fn prompt_parts(&self) -> Result, ChatSessionError> { + let part_count = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_prompt_part_count(self.session.as_ptr()) + }; + + (0..part_count) + .map(|index| { + let text = self.text_of(|session, out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_prompt_part_text( + session, index, out_length, + ) + })?; + let is_input = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_prompt_part_is_input( + self.session.as_ptr(), + index, + ) + }; + + Ok(if is_input { + ChatPromptPart::Input(text) + } else { + ChatPromptPart::Template(text) + }) + }) + .collect() + } + + /// The rendered prompt as one string, for [`crate::mtmd::MtmdContext::tokenize`], which + /// finds the media markers in it the way llama-server hands a multimodal prompt to mtmd. + /// + /// # Errors + /// + /// Returns [`ChatSessionError::SessionTextNotUtf8`] when the rendered prompt is not UTF-8. + pub fn prompt(&self) -> Result { + Ok(self + .prompt_parts()? + .iter() + .map(ChatPromptPart::text) + .collect()) + } + + /// Tokenizes the rendered prompt part by part: control tokens are recognized in the + /// template's own text but never in user input, and BOS is added once before the first part + /// when the vocabulary asks for it. + /// + /// # Errors + /// + /// Returns the [`ChatSessionError`] of reading or tokenizing a prompt part. + pub fn tokenize_prompt(&self) -> Result, ChatSessionError> { + let mut tokens = Vec::new(); + + for (part_index, part) in self.prompt_parts()?.iter().enumerate() { + let add_bos = if part_index == 0 { + AddBos::Always + } else { + AddBos::Never + }; + let (text, parse_special) = match part { + ChatPromptPart::Template(text) => (text, ParseSpecialTokens::Always), + ChatPromptPart::Input(text) => (text, ParseSpecialTokens::Never), + }; + + tokens.extend(self.model.str_to_token(text, add_bos, parse_special)?); + } + + Ok(tokens) + } + + /// # Errors + /// + /// Returns [`ChatSessionError::SessionTextNotUtf8`] when the generation prompt is not UTF-8. + pub fn generation_prompt(&self) -> Result { + self.text_of(|session, out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_generation_prompt(session, out_length) + }) + } + + #[must_use] + pub fn generation_prompt_tokens(&self) -> Vec { + let count = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_generation_prompt_token_count( + self.session.as_ptr(), + ) + }; + + (0..count) + .map(|index| { + LlamaToken::new(unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_generation_prompt_token( + self.session.as_ptr(), + index, + ) + }) + }) + .collect() + } + + /// # Errors + /// + /// Returns [`ChatSessionError::SessionTextNotUtf8`] when the grammar is not UTF-8. + pub fn grammar(&self) -> Result { + self.text_of(|session, out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_grammar(session, out_length) + }) + } + + #[must_use] + pub fn grammar_is_lazy(&self) -> bool { + unsafe { llama_cpp_bindings_sys::llama_rs_chat_session_grammar_lazy(self.session.as_ptr()) } + } + + /// # Errors + /// + /// Returns [`ChatSessionError::SessionTextNotUtf8`] when a trigger is not UTF-8, or + /// [`ChatSessionError::FfiStatus`] for a trigger type unknown to this binding. + pub fn grammar_triggers(&self) -> Result, ChatSessionError> { + let count = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_grammar_trigger_count( + self.session.as_ptr(), + ) + }; + + (0..count) + .map(|index| { + let trigger_type = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_grammar_trigger_type( + self.session.as_ptr(), + index, + ) + }; + let value = self.text_of(|session, out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_grammar_trigger_value( + session, index, out_length, + ) + })?; + + match trigger_type { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_TOKEN => { + Ok(ChatGrammarTrigger::Token(LlamaToken::new(unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_grammar_trigger_token( + self.session.as_ptr(), + index, + ) + }))) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_WORD => { + Ok(ChatGrammarTrigger::Word(value)) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN => { + Ok(ChatGrammarTrigger::Pattern(value)) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL => { + Ok(ChatGrammarTrigger::PatternFull(value)) + } + other => Err(crate::FfiStatusError { + operation: "llama_rs_chat_session_grammar_trigger_type", + code: i64::from(other), + } + .into()), + } + }) + .collect() + } + + #[must_use] + pub fn preserved_tokens(&self) -> Vec { + let count = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_preserved_token_count( + self.session.as_ptr(), + ) + }; + + (0..count) + .map(|index| { + LlamaToken::new(unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_preserved_token( + self.session.as_ptr(), + index, + ) + }) + }) + .collect() + } + + /// # Errors + /// + /// Returns [`ChatSessionError::SessionTextNotUtf8`] when the tag is not UTF-8. + pub fn thinking_start_tag(&self) -> Result { + self.text_of(|session, out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_thinking_start_tag(session, out_length) + }) + } + + /// # Errors + /// + /// Returns [`ChatSessionError::SessionTextNotUtf8`] when a tag is not UTF-8. + pub fn thinking_end_tags(&self) -> Result, ChatSessionError> { + self.texts_of( + unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_thinking_end_tag_count( + self.session.as_ptr(), + ) + }, + |session, index, out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_thinking_end_tag( + session, index, out_length, + ) + }, + ) + } + + /// # Errors + /// + /// Returns [`ChatSessionError::SessionTextNotUtf8`] when a stop is not UTF-8. + pub fn additional_stops(&self) -> Result, ChatSessionError> { + self.texts_of( + unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_additional_stop_count( + self.session.as_ptr(), + ) + }, + |session, index, out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_additional_stop( + session, index, out_length, + ) + }, + ) + } + + /// Whether the template this request rendered with can render tool calls. When it cannot, + /// [`Self::parse`] recognizes tool calls in the formats other templates use. + #[must_use] + pub const fn supports_tool_calls(&self) -> bool { + self.supports_tool_calls + } + + #[must_use] + pub const fn model(&self) -> &'model LlamaModel { + self.model + } + + /// The sampler that constrains generation to the request's tool calls, or `None` when the + /// request needs no grammar. A grammar that is not lazy starts after the generation prompt, + /// so it accepts the generation-prompt tokens first. + /// + /// # Errors + /// + /// Returns the [`ChatSessionError`] of reading the grammar, escaping a trigger word, building + /// the sampler, or accepting a generation-prompt token. + pub fn grammar_sampler(&self) -> Result, ChatSessionError> { + let grammar = self.grammar()?; + + if grammar.is_empty() { + return Ok(None); + } + + if !self.grammar_is_lazy() { + let mut sampler = LlamaSampler::grammar(self.model, &grammar, GRAMMAR_ROOT)?; + + for token in self.generation_prompt_tokens() { + sampler.accept(token)?; + } + + return Ok(Some(sampler)); + } + + let mut trigger_patterns = Vec::new(); + let mut trigger_tokens = Vec::new(); + + for trigger in self.grammar_triggers()? { + match trigger { + ChatGrammarTrigger::Token(token) => trigger_tokens.push(token), + ChatGrammarTrigger::Word(word) => trigger_patterns.push(regex_escaped(&word)?), + ChatGrammarTrigger::Pattern(pattern) => trigger_patterns.push(pattern), + ChatGrammarTrigger::PatternFull(pattern) => { + trigger_patterns.push(anchored_pattern(&pattern)); + } + } + } + + Ok(Some(LlamaSampler::grammar_lazy( + self.model, + &grammar, + GRAMMAR_ROOT, + &trigger_patterns, + &trigger_tokens, + )?)) + } + + fn parse_with_template( + &self, + output: &ChatOutput, + completeness: ChatOutputCompleteness, + ) -> Result { + let mut out_handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_parse( + self.session.as_ptr(), + output.text().as_ptr().cast::(), + output.text().len(), + output.byte_tokens().as_ptr(), + completeness == ChatOutputCompleteness::Partial, + &raw mut out_handle, + &raw mut out_error, + ) + }; + + chat_session_parse_status_to_result(status, out_handle, out_error, output, completeness) + } + + fn parse_duck_typed_tool_calls( + &self, + output: &ChatOutput, + completeness: ChatOutputCompleteness, + ) -> Result, ParseChatMessageError> { + for candidate in known_marker_candidates() { + if let ToolCallFormatOutcome::Parsed(tool_calls) = try_parse(output.text(), &candidate) + { + let before_tool_calls = output + .text() + .find(&candidate.open) + .map_or_else(|| output.text(), |open| &output.text()[..open]); + + return Ok(Some( + match self + .parse_with_template(&ChatOutput::from(before_tool_calls), completeness)? + { + ChatMessageParseOutcome::Recognized(prefix) => ParsedChatMessage::new( + prefix.content, + prefix.reasoning_content, + tool_calls, + ), + ChatMessageParseOutcome::Unrecognized(unparsed_prefix) => { + ParsedChatMessage::new(unparsed_prefix.text, String::new(), tool_calls) + } + }, + )); + } + } + + Ok(None) + } + + /// Parses the model's answer with the template's parser. When the template cannot render + /// tool calls, tool calls in the formats other templates use are recognized first. + /// + /// # Errors + /// + /// Returns [`ParseChatMessageError`] when llama.cpp fails while parsing; an answer that does + /// not match the template's format is [`ChatMessageParseOutcome::Unrecognized`]. + pub fn parse( + &self, + output: &ChatOutput, + completeness: ChatOutputCompleteness, + ) -> Result { + if !self.supports_tool_calls + && let Some(duck_typed) = self.parse_duck_typed_tool_calls(output, completeness)? + { + return Ok(ChatMessageParseOutcome::Recognized(duck_typed)); + } + + self.parse_with_template(output, completeness) + } + + /// Returns how the template's parser attributes the bytes of the model's answer to + /// reasoning, content, and tool calls. + /// + /// # Errors + /// + /// Returns [`ParseChatMessageError`] when llama.cpp fails while parsing. + pub fn classify( + &self, + output: &ChatOutput, + completeness: ChatOutputCompleteness, + ) -> Result { + let mut out_spans: *mut llama_cpp_bindings_sys::llama_rs_chat_spans = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_classify( + self.session.as_ptr(), + output.text().as_ptr().cast::(), + output.text().len(), + output.byte_tokens().as_ptr(), + completeness == ChatOutputCompleteness::Partial, + &raw mut out_spans, + &raw mut out_error, + ) + }; + + chat_session_classify_status_to_result(status, out_spans, out_error, completeness) + } +} + +impl Drop for ChatSession<'_> { + fn drop(&mut self) { + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_free( + self.session.as_ptr(), + &raw mut out_error, + ) + }; + + if let Err(destructor_failure) = chat_session_free_status_to_result(status, out_error) { + log::error!("{destructor_failure}"); + } + } +} + +#[cfg(test)] +mod tests { + use std::ffi::CStr; + use std::ffi::c_char; + use std::ptr; + + use serde_json::json; + + use super::anchored_pattern; + use super::chat_session_classify_status_to_result; + use super::chat_session_create_status_to_result; + use super::chat_session_free_status_to_result; + use super::chat_session_parse_status_to_result; + use super::chat_spans_free_status_to_result; + use super::messages_json; + use super::regex_escape_status_to_result; + use crate::chat_assistant_prefill::ChatAssistantPrefill; + use crate::chat_assistant_turn::ChatAssistantTurn; + use crate::chat_classification::ChatClassification; + use crate::chat_message_parse_outcome::ChatMessageParseOutcome; + use crate::chat_output::ChatOutput; + use crate::chat_output_completeness::ChatOutputCompleteness; + use crate::error::chat_session_error::ChatSessionError; + use crate::error::grammar_error::GrammarError; + use crate::error::parse_chat_message_error::ParseChatMessageError; + use crate::model::llama_chat_message::LlamaChatMessage; + use crate::raw_chat_message::RawChatMessage; + + type CreateRefusal = ( + llama_cpp_bindings_sys::llama_rs_chat_session_create_status, + fn(String) -> ChatSessionError, + ); + + fn cpp_message(message: &CStr) -> *mut c_char { + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(message.as_ptr()) } + } + + fn contract_violation( + operation: &'static str, + detail: &'static str, + ) -> crate::FfiContractError { + crate::FfiContractError { operation, detail } + } + + fn status_error(operation: &'static str, code: i64) -> crate::FfiStatusError { + crate::FfiStatusError { operation, code } + } + + fn parsed_messages(messages_json: &CStr) -> serde_json::Value { + serde_json::from_str(messages_json.to_str().unwrap()).unwrap() + } + + #[test] + fn creating_a_session_reports_each_null_argument_as_a_contract_violation() { + for (status, detail) in [ + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_TEMPLATES_ARG, + "was given a null templates argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_VOCAB_ARG, + "was given a null vocab argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_MESSAGES_JSON_ARG, + "was given a null messages_json argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_TOOLS_JSON_ARG, + "was given a null tools_json argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_SESSION_ARG, + "was given a null out_session argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_ERROR_ARG, + "was given a null out_error argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_OK, + "success status contained a null session", + ), + ] { + assert_eq!( + chat_session_create_status_to_result(status, ptr::null_mut(), ptr::null_mut()) + .err(), + Some(ChatSessionError::FfiContract(contract_violation( + "llama_rs_chat_session_create", + detail + ))) + ); + } + } + + #[test] + fn creating_a_session_reports_why_llama_cpp_refused_the_request() { + let refusals: [CreateRefusal; 6] = [ + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_MESSAGES_NOT_JSON, + |message| ChatSessionError::MessagesNotJson { message }, + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_MESSAGES_INVALID, + |message| ChatSessionError::MessagesInvalid { message }, + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_NOT_JSON, + |message| ChatSessionError::ToolsNotJson { message }, + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_INVALID, + |message| ChatSessionError::ToolsInvalid { message }, + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TEMPLATE_CANNOT_SERVE_REQUEST, + |message| ChatSessionError::TemplateCannotServeRequest { message }, + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, + |message| ChatSessionError::Reported { message }, + ), + ]; + + for (status, expected_error) in refusals { + assert_eq!( + chat_session_create_status_to_result( + status, + ptr::null_mut(), + cpp_message(c"the reason") + ) + .err(), + Some(expected_error("the reason".to_owned())) + ); + } + } + + #[test] + fn creating_a_session_reports_a_refusal_without_a_message_as_a_contract_violation() { + assert_eq!( + chat_session_create_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_INVALID, + ptr::null_mut(), + ptr::null_mut() + ) + .err(), + Some(ChatSessionError::FfiContract(contract_violation( + "llama_rs_chat_session_create", + "reported a failure without an error message" + ))) + ); + } + + #[test] + fn creating_a_session_reports_memory_exhaustion() { + assert_eq!( + [ + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_LLAMA_CPP_OUT_OF_MEMORY, + ] + .map(|status| chat_session_create_status_to_result( + status, + ptr::null_mut(), + ptr::null_mut() + ) + .err()), + [ + Some(ChatSessionError::NotEnoughMemory), + Some(ChatSessionError::LlamaCppOutOfMemory), + ] + ); + } + + #[test] + fn creating_a_session_preserves_an_unknown_status() { + assert_eq!( + chat_session_create_status_to_result(255, ptr::null_mut(), ptr::null_mut()).err(), + Some(ChatSessionError::FfiStatus(status_error( + "llama_rs_chat_session_create", + 255 + ))) + ); + } + + #[test] + fn freeing_a_session_maps_every_status() { + assert_eq!( + [ + chat_session_free_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_FREE_OK, + ptr::null_mut() + ), + chat_session_free_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_FREE_ERROR_STRING_ALLOCATION_FAILED, + ptr::null_mut() + ), + chat_session_free_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_FREE_LLAMA_CPP_OUT_OF_MEMORY, + ptr::null_mut() + ), + chat_session_free_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, + cpp_message(c"the destructor threw") + ), + chat_session_free_status_to_result(255, ptr::null_mut()), + ], + [ + Ok(()), + Err(ChatSessionError::NotEnoughMemory), + Err(ChatSessionError::LlamaCppOutOfMemory), + Err(ChatSessionError::Reported { + message: "the destructor threw".to_owned() + }), + Err(ChatSessionError::FfiStatus(status_error( + "llama_rs_chat_session_free", + 255 + ))), + ] + ); + } + + #[test] + fn parsing_an_answer_that_does_not_match_the_format_keeps_the_raw_answer() { + let output = ChatOutput::from("not the format"); + + assert_eq!( + chat_session_parse_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_OUTPUT_DOES_NOT_MATCH_FORMAT, + ptr::null_mut(), + cpp_message(c"the format differs"), + &output, + ChatOutputCompleteness::Finished, + ), + Ok(ChatMessageParseOutcome::Unrecognized(RawChatMessage { + text: "not the format".to_owned(), + completeness: ChatOutputCompleteness::Finished, + ffi_error_message: "the format differs".to_owned(), + })) + ); + } + + #[test] + fn parsing_an_answer_maps_every_failure_status() { + let output = ChatOutput::default(); + let parse = |status, out_error| { + chat_session_parse_status_to_result( + status, + ptr::null_mut(), + out_error, + &output, + ChatOutputCompleteness::Partial, + ) + .err() + }; + let contract = |detail| { + Some(ParseChatMessageError::FfiContract(contract_violation( + "llama_rs_chat_session_parse", + detail, + ))) + }; + + assert_eq!( + [ + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_OK, + ptr::null_mut() + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION, + cpp_message(c"the parser threw") + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_ERROR_STRING_ALLOCATION_FAILED, + ptr::null_mut() + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_OUT_OF_MEMORY, + ptr::null_mut() + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_SESSION_ARG, + ptr::null_mut() + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_TEXT_ARG, + ptr::null_mut() + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_BYTE_TOKENS_ARG, + ptr::null_mut() + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_HANDLE_ARG, + ptr::null_mut() + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_ERROR_ARG, + ptr::null_mut() + ), + parse(255, ptr::null_mut()), + ], + [ + contract("success status contained a null handle"), + Some(ParseChatMessageError::Reported { + message: "the parser threw".to_owned() + }), + Some(ParseChatMessageError::NotEnoughMemory), + Some(ParseChatMessageError::LlamaCppOutOfMemory), + contract("was given a null session argument"), + contract("was given a null text argument"), + contract("was given a null byte_tokens argument"), + contract("was given a null out_handle argument"), + contract("was given a null out_error argument"), + Some(ParseChatMessageError::FfiStatus(status_error( + "llama_rs_chat_session_parse", + 255 + ))), + ] + ); + } + + #[test] + fn classifying_an_answer_that_does_not_match_the_format_is_not_an_error() { + assert_eq!( + chat_session_classify_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_OUTPUT_DOES_NOT_MATCH_FORMAT, + ptr::null_mut(), + ptr::null_mut(), + ChatOutputCompleteness::Partial, + ), + Ok(ChatClassification::OutputDoesNotMatchFormat) + ); + } + + #[test] + fn classifying_an_answer_maps_every_failure_status() { + let classify = |status, out_error| { + chat_session_classify_status_to_result( + status, + ptr::null_mut(), + out_error, + ChatOutputCompleteness::Partial, + ) + .err() + }; + let contract = |detail| { + Some(ParseChatMessageError::FfiContract(contract_violation( + "llama_rs_chat_session_classify", + detail, + ))) + }; + + assert_eq!( + [ + classify( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_OK, + ptr::null_mut() + ), + classify( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_THREW_CXX_EXCEPTION, + cpp_message(c"the parser threw") + ), + classify( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_ERROR_STRING_ALLOCATION_FAILED, + ptr::null_mut() + ), + classify( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_OUT_OF_MEMORY, + ptr::null_mut() + ), + classify( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_SESSION_ARG, + ptr::null_mut() + ), + classify( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_TEXT_ARG, + ptr::null_mut() + ), + classify( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_BYTE_TOKENS_ARG, + ptr::null_mut() + ), + classify( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_SPANS_ARG, + ptr::null_mut() + ), + classify( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_ERROR_ARG, + ptr::null_mut() + ), + classify(255, ptr::null_mut()), + ], + [ + contract("success status contained null spans"), + Some(ParseChatMessageError::Reported { + message: "the parser threw".to_owned() + }), + Some(ParseChatMessageError::NotEnoughMemory), + Some(ParseChatMessageError::LlamaCppOutOfMemory), + contract("was given a null session argument"), + contract("was given a null text argument"), + contract("was given a null byte_tokens argument"), + contract("was given a null out_spans argument"), + contract("was given a null out_error argument"), + Some(ParseChatMessageError::FfiStatus(status_error( + "llama_rs_chat_session_classify", + 255 + ))), + ] + ); + } + + #[test] + fn freeing_classified_spans_maps_every_status() { + assert_eq!( + [ + chat_spans_free_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPANS_FREE_OK, + ptr::null_mut() + ), + chat_spans_free_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPANS_FREE_ERROR_STRING_ALLOCATION_FAILED, + ptr::null_mut() + ), + chat_spans_free_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPANS_FREE_LLAMA_CPP_OUT_OF_MEMORY, + ptr::null_mut() + ), + chat_spans_free_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPANS_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, + cpp_message(c"the destructor threw") + ), + chat_spans_free_status_to_result(255, ptr::null_mut()), + ], + [ + Ok(()), + Err(ParseChatMessageError::NotEnoughMemory), + Err(ParseChatMessageError::LlamaCppOutOfMemory), + Err(ParseChatMessageError::DestructorFailed { + message: "the destructor threw".to_owned() + }), + Err(ParseChatMessageError::FfiStatus(status_error( + "llama_rs_chat_spans_free", + 255 + ))), + ] + ); + } + + #[test] + fn escaping_a_trigger_word_maps_every_status() { + let escape = |status, out_string, out_error| { + regex_escape_status_to_result(status, out_string, out_error) + }; + let contract = |detail| { + Err(GrammarError::FfiContract(contract_violation( + "llama_rs_regex_escape", + detail, + ))) + }; + + assert_eq!( + [ + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_OK, + cpp_message(c"\\[TOOL\\]"), + ptr::null_mut() + ), + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_OK, + ptr::null_mut(), + ptr::null_mut() + ), + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + cpp_message(c"the escape threw") + ), + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_ERROR_STRING_ALLOCATION_FAILED, + ptr::null_mut(), + ptr::null_mut() + ), + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_OUT_OF_MEMORY, + ptr::null_mut(), + ptr::null_mut() + ), + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_TEXT_ARG, + ptr::null_mut(), + ptr::null_mut() + ), + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_OUT_STRING_ARG, + ptr::null_mut(), + ptr::null_mut() + ), + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_OUT_ERROR_ARG, + ptr::null_mut(), + ptr::null_mut() + ), + escape(255, ptr::null_mut(), ptr::null_mut()), + ], + [ + Ok("\\[TOOL\\]".to_owned()), + contract("success status contained a null string"), + Err(GrammarError::Reported { + message: "the escape threw".to_owned() + }), + Err(GrammarError::NotEnoughMemory), + Err(GrammarError::LlamaCppOutOfMemory), + contract("was given a null text argument"), + contract("was given a null out_string argument"), + contract("was given a null out_error argument"), + Err(GrammarError::FfiStatus(status_error( + "llama_rs_regex_escape", + 255 + ))), + ] + ); + } + + #[test] + fn a_full_match_pattern_is_anchored_at_both_ends_once() { + assert_eq!( + ["", "abc", "^abc", "abc$", "^abc$"].map(anchored_pattern), + ["^$", "^abc$", "^abc$", "^abc$", "^abc$"].map(str::to_owned) + ); + } + + #[test] + fn a_new_assistant_turn_adds_no_message() { + let messages = [LlamaChatMessage::new("user".to_owned(), "hi".to_owned()).unwrap()]; + + assert_eq!( + parsed_messages(&messages_json(&messages, &ChatAssistantTurn::New).unwrap()), + json!([{"role": "user", "content": "hi"}]) + ); + } + + #[test] + fn continuing_reasoning_appends_an_assistant_message_with_that_reasoning() { + assert_eq!( + parsed_messages( + &messages_json(&[], &ChatAssistantTurn::ContinueReasoning("hmm".to_owned())) + .unwrap() + ), + json!([{"role": "assistant", "content": "", "reasoning_content": "hmm"}]) + ); + } + + #[test] + fn continuing_content_appends_an_assistant_message_with_the_prefill() { + assert_eq!( + parsed_messages( + &messages_json( + &[], + &ChatAssistantTurn::ContinueContent(ChatAssistantPrefill { + reasoning_content: "done".to_owned(), + content: "The".to_owned(), + }) + ) + .unwrap() + ), + json!([{"role": "assistant", "content": "The", "reasoning_content": "done"}]) + ); + } +} diff --git a/llama-cpp-bindings/src/chat_session_options.rs b/llama-cpp-bindings/src/chat_session_options.rs new file mode 100644 index 000000000..d4a4835f1 --- /dev/null +++ b/llama-cpp-bindings/src/chat_session_options.rs @@ -0,0 +1,28 @@ +use crate::chat_assistant_turn::ChatAssistantTurn; +use crate::chat_tool_choice::ChatToolChoice; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ChatSessionOptions { + pub assistant_turn: ChatAssistantTurn, + pub enable_thinking: bool, + pub parallel_tool_calls: bool, + pub tool_choice: ChatToolChoice, +} + +impl From<&ChatSessionOptions> for llama_cpp_bindings_sys::llama_rs_chat_session_options { + fn from( + ChatSessionOptions { + assistant_turn, + enable_thinking, + parallel_tool_calls, + tool_choice, + }: &ChatSessionOptions, + ) -> Self { + Self { + assistant_turn: assistant_turn.into(), + enable_thinking: *enable_thinking, + parallel_tool_calls: *parallel_tool_calls, + tool_choice: (*tool_choice).into(), + } + } +} diff --git a/llama-cpp-bindings/src/chat_span.rs b/llama-cpp-bindings/src/chat_span.rs new file mode 100644 index 000000000..e61a214f8 --- /dev/null +++ b/llama-cpp-bindings/src/chat_span.rs @@ -0,0 +1,9 @@ +use std::ops::Range; + +use crate::chat_span_kind::ChatSpanKind; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ChatSpan { + pub kind: ChatSpanKind, + pub bytes: Range, +} diff --git a/llama-cpp-bindings/src/chat_span_kind.rs b/llama-cpp-bindings/src/chat_span_kind.rs new file mode 100644 index 000000000..952375a8e --- /dev/null +++ b/llama-cpp-bindings/src/chat_span_kind.rs @@ -0,0 +1,116 @@ +use crate::chat_piece_role::ChatPieceRole; +use crate::sampled_token_section::SampledTokenSection; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ChatSpanKind { + ReasoningBlock, + Reasoning, + Content, + ToolCall, + ToolCallMarker, +} + +impl ChatSpanKind { + #[must_use] + pub const fn piece_role(self) -> ChatPieceRole { + match self { + Self::ReasoningBlock => ChatPieceRole::Marker(SampledTokenSection::Reasoning), + Self::Reasoning => ChatPieceRole::Text(SampledTokenSection::Reasoning), + Self::Content => ChatPieceRole::Text(SampledTokenSection::Content), + Self::ToolCall => ChatPieceRole::Text(SampledTokenSection::ToolCall), + Self::ToolCallMarker => ChatPieceRole::Marker(SampledTokenSection::ToolCall), + } + } +} + +impl TryFrom for ChatSpanKind { + type Error = crate::FfiStatusError; + + fn try_from( + kind: llama_cpp_bindings_sys::llama_rs_chat_span_kind, + ) -> Result { + match kind { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_REASONING_BLOCK => { + Ok(Self::ReasoningBlock) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_REASONING => Ok(Self::Reasoning), + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_CONTENT => Ok(Self::Content), + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL => Ok(Self::ToolCall), + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL_MARKER => { + Ok(Self::ToolCallMarker) + } + other => Err(crate::FfiStatusError { + operation: "llama_rs_chat_spans_kind", + code: i64::from(other), + }), + } + } +} + +#[cfg(test)] +mod tests { + use super::ChatSpanKind; + use crate::chat_piece_role::ChatPieceRole; + use crate::sampled_token_section::SampledTokenSection; + + #[test] + fn text_spans_attribute_their_bytes_as_visible_text_of_their_section() { + assert_eq!( + [ + ChatSpanKind::Reasoning, + ChatSpanKind::Content, + ChatSpanKind::ToolCall + ] + .map(ChatSpanKind::piece_role), + [ + ChatPieceRole::Text(SampledTokenSection::Reasoning), + ChatPieceRole::Text(SampledTokenSection::Content), + ChatPieceRole::Text(SampledTokenSection::ToolCall), + ] + ); + } + + #[test] + fn marker_spans_attribute_their_bytes_as_markers_of_their_section() { + assert_eq!( + [ChatSpanKind::ReasoningBlock, ChatSpanKind::ToolCallMarker] + .map(ChatSpanKind::piece_role), + [ + ChatPieceRole::Marker(SampledTokenSection::Reasoning), + ChatPieceRole::Marker(SampledTokenSection::ToolCall), + ] + ); + } + + #[test] + fn every_wrapper_span_kind_maps_to_its_variant() { + assert_eq!( + [ + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_REASONING_BLOCK, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_REASONING, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_CONTENT, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL_MARKER, + ] + .map(ChatSpanKind::try_from), + [ + Ok(ChatSpanKind::ReasoningBlock), + Ok(ChatSpanKind::Reasoning), + Ok(ChatSpanKind::Content), + Ok(ChatSpanKind::ToolCall), + Ok(ChatSpanKind::ToolCallMarker), + ] + ); + } + + #[test] + fn an_unknown_wrapper_span_kind_is_a_status_error() { + assert_eq!( + ChatSpanKind::try_from(99), + Err(crate::FfiStatusError { + operation: "llama_rs_chat_spans_kind", + code: 99, + }) + ); + } +} diff --git a/llama-cpp-bindings/src/chat_spans.rs b/llama-cpp-bindings/src/chat_spans.rs new file mode 100644 index 000000000..9e8f55d6f --- /dev/null +++ b/llama-cpp-bindings/src/chat_spans.rs @@ -0,0 +1,248 @@ +use std::cmp::Reverse; +use std::ops::Range; + +use crate::chat_output_completeness::ChatOutputCompleteness; +use crate::chat_piece_role::ChatPieceRole; +use crate::chat_span::ChatSpan; +use crate::sampled_token_section::SampledTokenSection; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ChatSpans { + pub parsed_length: usize, + pub spans: Vec, + pub completeness: ChatOutputCompleteness, +} + +impl ChatSpans { + fn tagged_role_at(&self, byte: usize) -> Option { + self.spans + .iter() + .enumerate() + .filter(|(_order, span)| span.bytes.contains(&byte)) + .min_by_key(|(order, span)| (span.bytes.len(), Reverse(*order))) + .map(|(_order, span)| span.kind.piece_role()) + } + + fn untagged_marker_section(&self, byte: usize) -> Option { + let next = (byte..self.parsed_length) + .find_map(|later| self.tagged_role_at(later)) + .map(ChatPieceRole::section); + + if next.is_none() && self.completeness == ChatOutputCompleteness::Partial { + return None; + } + + let previous = (0..byte) + .rev() + .find_map(|earlier| self.tagged_role_at(earlier)) + .map(ChatPieceRole::section); + + Some(match (previous, next) { + (Some(SampledTokenSection::Reasoning), _) + | (_, Some(SampledTokenSection::Reasoning)) => SampledTokenSection::Reasoning, + (Some(SampledTokenSection::ToolCall), _) | (_, Some(SampledTokenSection::ToolCall)) => { + SampledTokenSection::ToolCall + } + _ => SampledTokenSection::Content, + }) + } + + /// Returns how llama.cpp's parse attributes the token whose text occupies `bytes` of the + /// output, or `None` while the parse has not settled the token yet. + #[must_use] + pub fn classify_token(&self, bytes: &Range) -> Option { + if self.completeness == ChatOutputCompleteness::Partial && bytes.end > self.parsed_length { + return None; + } + + let roles = bytes + .clone() + .map(|byte| self.tagged_role_at(byte)) + .collect::>>(); + + if let Some(text_role) = roles + .iter() + .flatten() + .find(|role| matches!(role, ChatPieceRole::Text(_))) + { + return Some(*text_role); + } + + match roles.first() { + Some(Some(marker_role)) => Some(*marker_role), + Some(None) | None => self + .untagged_marker_section(bytes.start) + .map(ChatPieceRole::Marker), + } + } +} + +#[cfg(test)] +mod tests { + use crate::chat_output_completeness::ChatOutputCompleteness; + use crate::chat_piece_role::ChatPieceRole; + use crate::chat_span::ChatSpan; + use crate::chat_span_kind::ChatSpanKind; + use crate::sampled_token_section::SampledTokenSection; + + use super::ChatSpans; + + fn span(kind: ChatSpanKind, start: usize, end: usize) -> ChatSpan { + ChatSpan { + kind, + bytes: start..end, + } + } + + fn reasoning_then_content(completeness: ChatOutputCompleteness) -> ChatSpans { + ChatSpans { + parsed_length: 30, + spans: vec![ + span(ChatSpanKind::Reasoning, 7, 12), + span(ChatSpanKind::Content, 22, 30), + ], + completeness, + } + } + + #[test] + fn a_token_inside_reasoning_text_is_visible_reasoning() { + assert_eq!( + reasoning_then_content(ChatOutputCompleteness::Partial).classify_token(&(7..12)), + Some(ChatPieceRole::Text(SampledTokenSection::Reasoning)) + ); + } + + #[test] + fn an_untagged_token_before_reasoning_opens_reasoning() { + assert_eq!( + reasoning_then_content(ChatOutputCompleteness::Partial).classify_token(&(0..7)), + Some(ChatPieceRole::Marker(SampledTokenSection::Reasoning)) + ); + } + + #[test] + fn an_untagged_token_after_reasoning_closes_reasoning() { + assert_eq!( + reasoning_then_content(ChatOutputCompleteness::Partial).classify_token(&(12..22)), + Some(ChatPieceRole::Marker(SampledTokenSection::Reasoning)) + ); + } + + #[test] + fn a_token_partly_in_content_text_is_visible_content() { + assert_eq!( + reasoning_then_content(ChatOutputCompleteness::Partial).classify_token(&(20..24)), + Some(ChatPieceRole::Text(SampledTokenSection::Content)) + ); + } + + #[test] + fn a_token_past_the_partial_parse_waits_for_more_output() { + assert_eq!( + reasoning_then_content(ChatOutputCompleteness::Partial).classify_token(&(28..31)), + None + ); + } + + #[test] + fn the_innermost_span_decides_a_byte_inside_nested_spans() { + let spans = ChatSpans { + parsed_length: 20, + spans: vec![ + span(ChatSpanKind::ToolCall, 0, 20), + span(ChatSpanKind::ToolCallMarker, 0, 6), + ], + completeness: ChatOutputCompleteness::Partial, + }; + + assert_eq!( + [ + spans.classify_token(&(0..6)), + spans.classify_token(&(6..10)) + ], + [ + Some(ChatPieceRole::Marker(SampledTokenSection::ToolCall)), + Some(ChatPieceRole::Text(SampledTokenSection::ToolCall)), + ] + ); + } + + #[test] + fn an_untagged_token_beside_a_tool_call_marks_the_tool_call() { + let spans = ChatSpans { + parsed_length: 20, + spans: vec![ + span(ChatSpanKind::Content, 0, 5), + span(ChatSpanKind::ToolCall, 10, 20), + ], + completeness: ChatOutputCompleteness::Partial, + }; + + assert_eq!( + spans.classify_token(&(5..10)), + Some(ChatPieceRole::Marker(SampledTokenSection::ToolCall)) + ); + } + + #[test] + fn an_untagged_partial_tail_after_content_waits_for_what_follows() { + let spans = ChatSpans { + parsed_length: 8, + spans: vec![span(ChatSpanKind::Content, 0, 5)], + completeness: ChatOutputCompleteness::Partial, + }; + + assert_eq!(spans.classify_token(&(5..8)), None); + } + + #[test] + fn an_untagged_partial_tail_after_reasoning_waits_for_what_follows() { + let spans = ChatSpans { + parsed_length: 13, + spans: vec![span(ChatSpanKind::Reasoning, 0, 12)], + completeness: ChatOutputCompleteness::Partial, + }; + + assert_eq!(spans.classify_token(&(12..13)), None); + } + + #[test] + fn an_untagged_tail_of_a_finished_parse_after_content_is_a_content_marker() { + let spans = ChatSpans { + parsed_length: 8, + spans: vec![span(ChatSpanKind::Content, 0, 5)], + completeness: ChatOutputCompleteness::Finished, + }; + + assert_eq!( + spans.classify_token(&(5..8)), + Some(ChatPieceRole::Marker(SampledTokenSection::Content)) + ); + } + + #[test] + fn a_token_without_text_takes_the_role_of_its_position() { + assert_eq!( + reasoning_then_content(ChatOutputCompleteness::Partial).classify_token(&(12..12)), + Some(ChatPieceRole::Marker(SampledTokenSection::Reasoning)) + ); + } + + #[test] + fn a_reasoning_block_byte_outside_reasoning_text_is_a_reasoning_marker() { + let spans = ChatSpans { + parsed_length: 20, + spans: vec![ + span(ChatSpanKind::ReasoningBlock, 0, 20), + span(ChatSpanKind::Reasoning, 7, 12), + ], + completeness: ChatOutputCompleteness::Partial, + }; + + assert_eq!( + spans.classify_token(&(12..20)), + Some(ChatPieceRole::Marker(SampledTokenSection::Reasoning)) + ); + } +} diff --git a/llama-cpp-bindings/src/chat_template_tool_calls.rs b/llama-cpp-bindings/src/chat_template_tool_calls.rs index a1b5010e9..4cad316e3 100644 --- a/llama-cpp-bindings/src/chat_template_tool_calls.rs +++ b/llama-cpp-bindings/src/chat_template_tool_calls.rs @@ -1,11 +1,8 @@ -pub mod detect; pub mod gemma4_call_block; pub mod glm47_key_value_tags; pub mod known_marker_candidates; pub mod mistral3_arrow_args; pub mod qwen3_json_inside_tool_call; pub mod qwen_xml_tags; -pub mod tool_call_format_registration; -pub use detect::detect; pub use known_marker_candidates::known_marker_candidates; diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/detect.rs b/llama-cpp-bindings/src/chat_template_tool_calls/detect.rs deleted file mode 100644 index 422e4ea7d..000000000 --- a/llama-cpp-bindings/src/chat_template_tool_calls/detect.rs +++ /dev/null @@ -1,47 +0,0 @@ -use llama_cpp_bindings_types::ToolCallMarkers; - -use crate::chat_template_tool_calls::tool_call_format_registration::ToolCallFormatRegistration; - -#[must_use] -pub fn detect(template: &str) -> Option { - ToolCallFormatRegistration::KNOWN - .iter() - .find_map(|registration| (registration.detect)(template)) -} - -#[cfg(test)] -mod tests { - use super::detect; - use crate::chat_template_tool_calls::gemma4_call_block::Gemma4CallBlockFormat; - use crate::chat_template_tool_calls::mistral3_arrow_args::Mistral3ArrowArgsFormat; - use crate::chat_template_tool_calls::qwen_xml_tags::QwenXmlTagsFormat; - - #[test] - fn detects_gemma4_call_block_format() { - let template = "{{- '<|tool_call>call:' + function['name'] + '{' -}}"; - let markers = detect(template).expect("must dispatch to Gemma 4"); - - assert_eq!(markers, Gemma4CallBlockFormat::markers()); - } - - #[test] - fn detects_mistral3_arrow_args_format() { - let template = "{{- name + '[ARGS]' + arguments }}"; - let markers = detect(template).expect("must dispatch to Mistral 3"); - - assert_eq!(markers, Mistral3ArrowArgsFormat::markers()); - } - - #[test] - fn detects_qwen_xml_tags_format() { - let template = "{{- '\\n\\n' }}"; - let markers = detect(template).expect("must dispatch to Qwen XML tags"); - - assert_eq!(markers, QwenXmlTagsFormat::markers()); - } - - #[test] - fn returns_none_when_no_known_format_matches() { - assert!(detect("plain unrelated template").is_none()); - } -} diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/gemma4_call_block.rs b/llama-cpp-bindings/src/chat_template_tool_calls/gemma4_call_block.rs index 978ddeea3..add745253 100644 --- a/llama-cpp-bindings/src/chat_template_tool_calls/gemma4_call_block.rs +++ b/llama-cpp-bindings/src/chat_template_tool_calls/gemma4_call_block.rs @@ -6,8 +6,6 @@ use llama_cpp_bindings_types::ToolCallValueQuote; pub struct Gemma4CallBlockFormat; impl Gemma4CallBlockFormat { - const TEMPLATE_FINGERPRINT: &'static str = "'<|tool_call>call:'"; - #[must_use] pub fn markers() -> ToolCallMarkers { ToolCallMarkers { @@ -22,58 +20,4 @@ impl Gemma4CallBlockFormat { }), } } - - #[must_use] - pub fn detect(template: &str) -> Option { - if !template.contains(Self::TEMPLATE_FINGERPRINT) { - return None; - } - Some(Self::markers()) - } -} - -#[cfg(test)] -mod tests { - use llama_cpp_bindings_types::ToolCallArgsShape; - - use super::Gemma4CallBlockFormat; - - #[test] - fn detects_gemma4_template_with_tool_call_call_literal() { - use llama_cpp_bindings_types::PairedQuoteShape; - use llama_cpp_bindings_types::ToolCallValueQuote; - - let template = "...{{- '<|tool_call>call:' + function['name'] + '{' -}}..."; - let markers = - Gemma4CallBlockFormat::detect(template).expect("Gemma 4 template must be detected"); - - assert_eq!(markers.open, "<|tool_call>call:"); - assert_eq!(markers.close, "}"); - assert_eq!( - markers.args_shape, - ToolCallArgsShape::PairedQuote(PairedQuoteShape { - name_args_separator: "{".to_owned(), - value_quote: ToolCallValueQuote { - open: "<|\"|>".to_owned(), - close: "<|\"|>".to_owned(), - }, - }) - ); - } - - #[test] - fn returns_none_for_template_without_fingerprint() { - assert!(Gemma4CallBlockFormat::detect("just some plain template body").is_none()); - } - - #[test] - fn returns_none_for_empty_template() { - assert!(Gemma4CallBlockFormat::detect("").is_none()); - } - - #[test] - fn returns_none_when_fingerprint_substring_appears_without_jinja_apostrophes() { - let template = "doc explaining the <|tool_call>call: format in prose, not as a literal"; - assert!(Gemma4CallBlockFormat::detect(template).is_none()); - } } diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/glm47_key_value_tags.rs b/llama-cpp-bindings/src/chat_template_tool_calls/glm47_key_value_tags.rs index 5e1627f2a..437d64b50 100644 --- a/llama-cpp-bindings/src/chat_template_tool_calls/glm47_key_value_tags.rs +++ b/llama-cpp-bindings/src/chat_template_tool_calls/glm47_key_value_tags.rs @@ -5,8 +5,6 @@ use llama_cpp_bindings_types::ToolCallMarkers; pub struct Glm47KeyValueTagsFormat; impl Glm47KeyValueTagsFormat { - const TEMPLATE_FINGERPRINT: &'static str = ""; - #[must_use] pub fn markers() -> ToolCallMarkers { ToolCallMarkers { @@ -20,49 +18,4 @@ impl Glm47KeyValueTagsFormat { }), } } - - #[must_use] - pub fn detect(template: &str) -> Option { - if !template.contains(Self::TEMPLATE_FINGERPRINT) { - return None; - } - Some(Self::markers()) - } -} - -#[cfg(test)] -mod tests { - use llama_cpp_bindings_types::KeyValueXmlTagsShape; - use llama_cpp_bindings_types::ToolCallArgsShape; - - use super::Glm47KeyValueTagsFormat; - - #[test] - fn detects_glm47_template_with_arg_key_literal() { - let template = "{{- '' + tool_call.name }}{% for k, v in args.items() %}{{ k }}{{ v }}{% endfor %}"; - let markers = - Glm47KeyValueTagsFormat::detect(template).expect("GLM-4.7 template must be detected"); - - assert_eq!(markers.open, ""); - assert_eq!(markers.close, ""); - assert_eq!( - markers.args_shape, - ToolCallArgsShape::KeyValueXmlTags(KeyValueXmlTagsShape { - key_open: "".to_owned(), - key_close: "".to_owned(), - value_open: "".to_owned(), - value_close: "".to_owned(), - }) - ); - } - - #[test] - fn returns_none_for_template_without_fingerprint() { - assert!(Glm47KeyValueTagsFormat::detect("just some plain template body").is_none()); - } - - #[test] - fn returns_none_for_empty_template() { - assert!(Glm47KeyValueTagsFormat::detect("").is_none()); - } } diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/known_marker_candidates.rs b/llama-cpp-bindings/src/chat_template_tool_calls/known_marker_candidates.rs index a37c0fc4b..f9272dc99 100644 --- a/llama-cpp-bindings/src/chat_template_tool_calls/known_marker_candidates.rs +++ b/llama-cpp-bindings/src/chat_template_tool_calls/known_marker_candidates.rs @@ -1,45 +1,20 @@ use llama_cpp_bindings_types::ToolCallMarkers; -use crate::chat_template_tool_calls::tool_call_format_registration::ToolCallFormatRegistration; - +use crate::chat_template_tool_calls::gemma4_call_block::Gemma4CallBlockFormat; +use crate::chat_template_tool_calls::glm47_key_value_tags::Glm47KeyValueTagsFormat; +use crate::chat_template_tool_calls::mistral3_arrow_args::Mistral3ArrowArgsFormat; +use crate::chat_template_tool_calls::qwen_xml_tags::QwenXmlTagsFormat; +use crate::chat_template_tool_calls::qwen3_json_inside_tool_call::Qwen3JsonInsideToolCallFormat; + +/// Every chat-template tool-call format this crate knows, ordered so that the more restrictive +/// shapes are tried before the ones whose separators could greedily match them. #[must_use] pub fn known_marker_candidates() -> Vec { - ToolCallFormatRegistration::KNOWN - .iter() - .map(|registration| (registration.markers)()) - .collect() -} - -#[cfg(test)] -mod tests { - use std::collections::HashSet; - - use llama_cpp_bindings_types::ToolCallArgsShape; - - use super::known_marker_candidates; - use crate::chat_template_tool_calls::tool_call_format_registration::ToolCallFormatRegistration; - - #[test] - fn known_marker_candidates_returns_one_per_registered_shape() { - let candidates = known_marker_candidates(); - - assert_eq!(candidates.len(), ToolCallFormatRegistration::KNOWN.len()); - - let shape_discriminants: HashSet<&'static str> = candidates - .iter() - .map(|markers| match &markers.args_shape { - ToolCallArgsShape::BracketedJson(_) => "BracketedJson", - ToolCallArgsShape::JsonObject(_) => "JsonObject", - ToolCallArgsShape::KeyValueXmlTags(_) => "KeyValueXmlTags", - ToolCallArgsShape::PairedQuote(_) => "PairedQuote", - ToolCallArgsShape::XmlTags(_) => "XmlTags", - }) - .collect(); - - assert_eq!( - shape_discriminants.len(), - candidates.len(), - "duplicate shape discriminants in known_marker_candidates: {shape_discriminants:?}" - ); - } + vec![ + Qwen3JsonInsideToolCallFormat::markers(), + QwenXmlTagsFormat::markers(), + Glm47KeyValueTagsFormat::markers(), + Mistral3ArrowArgsFormat::markers(), + Gemma4CallBlockFormat::markers(), + ] } diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/mistral3_arrow_args.rs b/llama-cpp-bindings/src/chat_template_tool_calls/mistral3_arrow_args.rs index f930eb9ea..d2d9779be 100644 --- a/llama-cpp-bindings/src/chat_template_tool_calls/mistral3_arrow_args.rs +++ b/llama-cpp-bindings/src/chat_template_tool_calls/mistral3_arrow_args.rs @@ -5,8 +5,6 @@ use llama_cpp_bindings_types::ToolCallMarkers; pub struct Mistral3ArrowArgsFormat; impl Mistral3ArrowArgsFormat { - const TEMPLATE_FINGERPRINT: &'static str = "'[ARGS]'"; - #[must_use] pub fn markers() -> ToolCallMarkers { ToolCallMarkers { @@ -17,52 +15,4 @@ impl Mistral3ArrowArgsFormat { }), } } - - #[must_use] - pub fn detect(template: &str) -> Option { - if !template.contains(Self::TEMPLATE_FINGERPRINT) { - return None; - } - Some(Self::markers()) - } -} - -#[cfg(test)] -mod tests { - use llama_cpp_bindings_types::BracketedJsonShape; - use llama_cpp_bindings_types::ToolCallArgsShape; - - use super::Mistral3ArrowArgsFormat; - - #[test] - fn detects_mistral3_template_with_args_literal() { - let template = "...{{- name + '[ARGS]' + arguments }}..."; - let markers = - Mistral3ArrowArgsFormat::detect(template).expect("Mistral 3 template must be detected"); - - assert_eq!(markers.open, "[TOOL_CALLS]"); - assert_eq!(markers.close, ""); - assert_eq!( - markers.args_shape, - ToolCallArgsShape::BracketedJson(BracketedJsonShape { - name_args_separator: "[ARGS]".to_owned(), - }) - ); - } - - #[test] - fn returns_none_for_template_without_fingerprint() { - assert!(Mistral3ArrowArgsFormat::detect("just some plain template body").is_none()); - } - - #[test] - fn returns_none_for_empty_template() { - assert!(Mistral3ArrowArgsFormat::detect("").is_none()); - } - - #[test] - fn returns_none_when_fingerprint_substring_appears_without_jinja_apostrophes() { - let template = "doc text mentioning the [ARGS] tag without quoting it as a literal"; - assert!(Mistral3ArrowArgsFormat::detect(template).is_none()); - } } diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/qwen3_json_inside_tool_call.rs b/llama-cpp-bindings/src/chat_template_tool_calls/qwen3_json_inside_tool_call.rs index f3b799a29..69ebc0c83 100644 --- a/llama-cpp-bindings/src/chat_template_tool_calls/qwen3_json_inside_tool_call.rs +++ b/llama-cpp-bindings/src/chat_template_tool_calls/qwen3_json_inside_tool_call.rs @@ -5,9 +5,6 @@ use llama_cpp_bindings_types::ToolCallMarkers; pub struct Qwen3JsonInsideToolCallFormat; impl Qwen3JsonInsideToolCallFormat { - const TEMPLATE_FINGERPRINT_OPEN: &'static str = "'\\n{\"name\": \"'"; - const TEMPLATE_FINGERPRINT_ARGS_JOIN: &'static str = "'\", \"arguments\": '"; - #[must_use] pub fn markers() -> ToolCallMarkers { ToolCallMarkers { @@ -19,65 +16,4 @@ impl Qwen3JsonInsideToolCallFormat { }), } } - - #[must_use] - pub fn detect(template: &str) -> Option { - if !template.contains(Self::TEMPLATE_FINGERPRINT_OPEN) { - return None; - } - if !template.contains(Self::TEMPLATE_FINGERPRINT_ARGS_JOIN) { - return None; - } - Some(Self::markers()) - } -} - -#[cfg(test)] -mod tests { - use llama_cpp_bindings_types::JsonObjectShape; - use llama_cpp_bindings_types::ToolCallArgsShape; - - use super::Qwen3JsonInsideToolCallFormat; - - #[test] - fn detects_qwen3_json_inside_tool_call_template() { - let template = "{{- '\\n{\"name\": \"' + tool_call.name + '\", \"arguments\": ' + (tool_call.arguments | tojson) + '}\\n' -}}"; - let markers = Qwen3JsonInsideToolCallFormat::detect(template) - .expect("Qwen 3 template must be detected"); - - assert_eq!(markers.open, ""); - assert_eq!(markers.close, ""); - assert_eq!( - markers.args_shape, - ToolCallArgsShape::JsonObject(JsonObjectShape { - name_field: "name".to_owned(), - arguments_field: "arguments".to_owned(), - }) - ); - } - - #[test] - fn returns_none_for_template_without_fingerprint() { - assert!(Qwen3JsonInsideToolCallFormat::detect("just some plain template body").is_none()); - } - - #[test] - fn returns_none_for_empty_template() { - assert!(Qwen3JsonInsideToolCallFormat::detect("").is_none()); - } - - #[test] - fn returns_none_when_only_open_fingerprint_present() { - let template = "{{- '\\n{\"name\": \"' + tool_call.name + ..."; - assert!( - Qwen3JsonInsideToolCallFormat::detect(template).is_none(), - "open fingerprint alone must not match (Qwen3-Embedding-style false positive)", - ); - } - - #[test] - fn returns_none_when_only_args_join_fingerprint_present() { - let template = "some text '\", \"arguments\": ' more text"; - assert!(Qwen3JsonInsideToolCallFormat::detect(template).is_none()); - } } diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/qwen_xml_tags.rs b/llama-cpp-bindings/src/chat_template_tool_calls/qwen_xml_tags.rs index bb97c750e..7211a221b 100644 --- a/llama-cpp-bindings/src/chat_template_tool_calls/qwen_xml_tags.rs +++ b/llama-cpp-bindings/src/chat_template_tool_calls/qwen_xml_tags.rs @@ -5,8 +5,6 @@ use llama_cpp_bindings_types::XmlTagsShape; pub struct QwenXmlTagsFormat; impl QwenXmlTagsFormat { - const TEMPLATE_FINGERPRINT: &'static str = " ToolCallMarkers { ToolCallMarkers { @@ -20,55 +18,4 @@ impl QwenXmlTagsFormat { }), } } - - #[must_use] - pub fn detect(template: &str) -> Option { - if !template.contains(Self::TEMPLATE_FINGERPRINT) { - return None; - } - Some(Self::markers()) - } -} - -#[cfg(test)] -mod tests { - use llama_cpp_bindings_types::ToolCallArgsShape; - use llama_cpp_bindings_types::XmlTagsShape; - - use super::QwenXmlTagsFormat; - - #[test] - fn detects_qwen_xml_template_with_function_tag_literal() { - let template = "{{- '\\n\\n' }}"; - let markers = - QwenXmlTagsFormat::detect(template).expect("Qwen XML template must be detected"); - - assert_eq!(markers.open, ""); - assert_eq!(markers.close, ""); - assert_eq!( - markers.args_shape, - ToolCallArgsShape::XmlTags(XmlTagsShape { - function_open_prefix: "".to_owned(), - parameter_open_prefix: "".to_owned(), - }) - ); - } - - #[test] - fn returns_none_for_template_without_fingerprint() { - assert!(QwenXmlTagsFormat::detect("just some plain template body").is_none()); - } - - #[test] - fn returns_none_for_empty_template() { - assert!(QwenXmlTagsFormat::detect("").is_none()); - } - - #[test] - fn detects_qwen_xml_template_with_concatenated_string_literal() { - let template = "{{- '\\n\\n\\n\\n' }}"; - assert!(QwenXmlTagsFormat::detect(template).is_some()); - } } diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/tool_call_format_registration.rs b/llama-cpp-bindings/src/chat_template_tool_calls/tool_call_format_registration.rs deleted file mode 100644 index c702b6d2e..000000000 --- a/llama-cpp-bindings/src/chat_template_tool_calls/tool_call_format_registration.rs +++ /dev/null @@ -1,40 +0,0 @@ -use llama_cpp_bindings_types::ToolCallMarkers; - -use crate::chat_template_tool_calls::gemma4_call_block::Gemma4CallBlockFormat; -use crate::chat_template_tool_calls::glm47_key_value_tags::Glm47KeyValueTagsFormat; -use crate::chat_template_tool_calls::mistral3_arrow_args::Mistral3ArrowArgsFormat; -use crate::chat_template_tool_calls::qwen_xml_tags::QwenXmlTagsFormat; -use crate::chat_template_tool_calls::qwen3_json_inside_tool_call::Qwen3JsonInsideToolCallFormat; - -pub struct ToolCallFormatRegistration { - pub detect: fn(&str) -> Option, - pub markers: fn() -> ToolCallMarkers, -} - -impl ToolCallFormatRegistration { - /// Every chat-template tool-call format this crate knows, ordered so that the more - /// restrictive shapes are tried before the ones whose separators could greedily - /// match them. - pub const KNOWN: &'static [Self] = &[ - Self { - detect: Qwen3JsonInsideToolCallFormat::detect, - markers: Qwen3JsonInsideToolCallFormat::markers, - }, - Self { - detect: QwenXmlTagsFormat::detect, - markers: QwenXmlTagsFormat::markers, - }, - Self { - detect: Glm47KeyValueTagsFormat::detect, - markers: Glm47KeyValueTagsFormat::markers, - }, - Self { - detect: Mistral3ArrowArgsFormat::detect, - markers: Mistral3ArrowArgsFormat::markers, - }, - Self { - detect: Gemma4CallBlockFormat::detect, - markers: Gemma4CallBlockFormat::markers, - }, - ]; -} diff --git a/llama-cpp-bindings/src/chat_templates.rs b/llama-cpp-bindings/src/chat_templates.rs new file mode 100644 index 000000000..7d720337c --- /dev/null +++ b/llama-cpp-bindings/src/chat_templates.rs @@ -0,0 +1,287 @@ +use std::ffi::c_char; +use std::ptr; +use std::ptr::NonNull; + +use llama_cpp_ffi_status::read_and_free_cpp_string; + +use crate::error::chat_templates_error::ChatTemplatesError; +use crate::model::LlamaModel; + +fn chat_templates_init_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_chat_templates_init_status, + out_templates: llama_cpp_bindings_sys::llama_rs_chat_templates_handle, + out_error: *mut c_char, +) -> Result, ChatTemplatesError> { + match status { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_OK => NonNull::new(out_templates) + .ok_or_else(|| { + crate::FfiContractError { + operation: "llama_rs_chat_templates_init", + detail: "success status contained a null templates handle", + } + .into() + }), + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_MODEL_HAS_NO_CHAT_TEMPLATE => { + Err(ChatTemplatesError::ModelHasNoChatTemplate) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_ERROR_STRING_ALLOCATION_FAILED => { + Err(ChatTemplatesError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_LLAMA_CPP_OUT_OF_MEMORY => { + Err(ChatTemplatesError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + "llama_rs_chat_templates_init", + "reported a thrown C++ exception without an error message", + ) + }?; + Err(ChatTemplatesError::Reported { message }) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_MODEL_ARG => { + Err(crate::FfiContractError { + operation: "llama_rs_chat_templates_init", + detail: "was given a null model argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_TEMPLATES_ARG => { + Err(crate::FfiContractError { + operation: "llama_rs_chat_templates_init", + detail: "was given a null out_templates argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_ERROR_ARG => { + Err(crate::FfiContractError { + operation: "llama_rs_chat_templates_init", + detail: "was given a null out_error argument", + } + .into()) + } + other => Err(crate::FfiStatusError { + operation: "llama_rs_chat_templates_init", + code: i64::from(other), + } + .into()), + } +} + +fn chat_templates_free_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_chat_templates_free_status, + out_error: *mut c_char, +) -> Result<(), ChatTemplatesError> { + match status { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_FREE_OK => Ok(()), + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_FREE_ERROR_STRING_ALLOCATION_FAILED => { + Err(ChatTemplatesError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_FREE_LLAMA_CPP_OUT_OF_MEMORY => { + Err(ChatTemplatesError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + "llama_rs_chat_templates_free", + "reported a thrown C++ exception without an error message", + ) + }?; + Err(ChatTemplatesError::Reported { message }) + } + other => Err(crate::FfiStatusError { + operation: "llama_rs_chat_templates_free", + code: i64::from(other), + } + .into()), + } +} + +/// The chat templates of a model as llama.cpp prepares them once per model: with the model's +/// BOS and EOS, the template variant for tool calls, and llama.cpp's template fixes applied. +#[derive(Debug)] +pub struct ChatTemplates { + templates: NonNull, +} + +unsafe impl Send for ChatTemplates {} +unsafe impl Sync for ChatTemplates {} + +impl ChatTemplates { + /// # Errors + /// + /// Returns [`ChatTemplatesError::ModelHasNoChatTemplate`] for a model without a chat template, + /// or another [`ChatTemplatesError`] when llama.cpp cannot prepare the templates. + pub fn new(model: &LlamaModel) -> Result { + let mut out_templates: llama_cpp_bindings_sys::llama_rs_chat_templates_handle = + ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_templates_init( + model.model.as_ptr(), + &raw mut out_templates, + &raw mut out_error, + ) + }; + + chat_templates_init_status_to_result(status, out_templates, out_error) + .map(|templates| Self { templates }) + } + + #[must_use] + pub const fn as_ptr(&self) -> llama_cpp_bindings_sys::llama_rs_chat_templates_handle { + self.templates.as_ptr() + } + + /// Whether the template a tool call request renders with can render tool calls, as llama.cpp + /// detects it from the template program. + #[must_use] + pub fn supports_tool_calls(&self) -> bool { + unsafe { + llama_cpp_bindings_sys::llama_rs_chat_templates_supports_tool_calls( + self.templates.as_ptr(), + ) + } + } +} + +impl Drop for ChatTemplates { + fn drop(&mut self) { + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_templates_free( + self.templates.as_ptr(), + &raw mut out_error, + ) + }; + + if let Err(destructor_failure) = chat_templates_free_status_to_result(status, out_error) { + log::error!("{destructor_failure}"); + } + } +} + +#[cfg(test)] +mod tests { + use std::ffi::CStr; + use std::ffi::c_char; + use std::ptr; + + use super::chat_templates_free_status_to_result; + use super::chat_templates_init_status_to_result; + use crate::error::chat_templates_error::ChatTemplatesError; + + fn cpp_message(message: &CStr) -> *mut c_char { + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(message.as_ptr()) } + } + + fn init_error( + status: llama_cpp_bindings_sys::llama_rs_chat_templates_init_status, + out_error: *mut c_char, + ) -> Option { + chat_templates_init_status_to_result(status, ptr::null_mut(), out_error).err() + } + + const fn contract(detail: &'static str) -> ChatTemplatesError { + ChatTemplatesError::FfiContract(crate::FfiContractError { + operation: "llama_rs_chat_templates_init", + detail, + }) + } + + #[test] + fn preparing_templates_maps_every_failure_status() { + assert_eq!( + [ + init_error( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_OK, + ptr::null_mut() + ), + init_error( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_MODEL_HAS_NO_CHAT_TEMPLATE, + ptr::null_mut() + ), + init_error( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_ERROR_STRING_ALLOCATION_FAILED, + ptr::null_mut() + ), + init_error( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_LLAMA_CPP_OUT_OF_MEMORY, + ptr::null_mut() + ), + init_error( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + cpp_message(c"the template threw") + ), + init_error( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_MODEL_ARG, + ptr::null_mut() + ), + init_error( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_TEMPLATES_ARG, + ptr::null_mut() + ), + init_error( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_ERROR_ARG, + ptr::null_mut() + ), + init_error(255, ptr::null_mut()), + ], + [ + Some(contract("success status contained a null templates handle")), + Some(ChatTemplatesError::ModelHasNoChatTemplate), + Some(ChatTemplatesError::NotEnoughMemory), + Some(ChatTemplatesError::LlamaCppOutOfMemory), + Some(ChatTemplatesError::Reported { + message: "the template threw".to_owned() + }), + Some(contract("was given a null model argument")), + Some(contract("was given a null out_templates argument")), + Some(contract("was given a null out_error argument")), + Some(ChatTemplatesError::FfiStatus(crate::FfiStatusError { + operation: "llama_rs_chat_templates_init", + code: 255, + })), + ] + ); + } + + #[test] + fn releasing_templates_maps_every_status() { + assert_eq!( + [ + chat_templates_free_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_FREE_OK, + ptr::null_mut() + ), + chat_templates_free_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_FREE_ERROR_STRING_ALLOCATION_FAILED, + ptr::null_mut() + ), + chat_templates_free_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_FREE_LLAMA_CPP_OUT_OF_MEMORY, + ptr::null_mut() + ), + chat_templates_free_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, + cpp_message(c"the destructor threw") + ), + chat_templates_free_status_to_result(255, ptr::null_mut()), + ], + [ + Ok(()), + Err(ChatTemplatesError::NotEnoughMemory), + Err(ChatTemplatesError::LlamaCppOutOfMemory), + Err(ChatTemplatesError::Reported { + message: "the destructor threw".to_owned() + }), + Err(ChatTemplatesError::FfiStatus(crate::FfiStatusError { + operation: "llama_rs_chat_templates_free", + code: 255, + })), + ] + ); + } +} diff --git a/llama-cpp-bindings/src/chat_tool_choice.rs b/llama-cpp-bindings/src/chat_tool_choice.rs new file mode 100644 index 000000000..fb09fad86 --- /dev/null +++ b/llama-cpp-bindings/src/chat_tool_choice.rs @@ -0,0 +1,16 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ChatToolChoice { + Auto, + None, + Required, +} + +impl From for llama_cpp_bindings_sys::llama_rs_chat_tool_choice { + fn from(tool_choice: ChatToolChoice) -> Self { + match tool_choice { + ChatToolChoice::Auto => llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOL_CHOICE_AUTO, + ChatToolChoice::None => llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOL_CHOICE_NONE, + ChatToolChoice::Required => llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOL_CHOICE_REQUIRED, + } + } +} diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index 1ae604972..58089f7da 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -578,7 +578,6 @@ impl<'model> LlamaContext<'model> { unsafe { llama_cpp_bindings_sys::llama_n_ctx_seq(self.context.as_ptr()) } } - #[expect(unsafe_code, reason = "required for FFI abort callback registration")] pub fn set_abort_flag(&mut self, flag: Arc) { let raw_ptr = Arc::as_ptr(&flag) as *mut c_void; self.abort_flag = Some(flag); @@ -592,7 +591,6 @@ impl<'model> LlamaContext<'model> { } } - #[expect(unsafe_code, reason = "required for FFI abort callback deregistration")] pub fn clear_abort_callback(&mut self) { self.abort_flag = None; @@ -605,12 +603,10 @@ impl<'model> LlamaContext<'model> { } } - #[expect(unsafe_code, reason = "required for FFI synchronization call")] pub fn synchronize(&self) { unsafe { llama_cpp_bindings_sys::llama_synchronize(self.context.as_ptr()) } } - #[expect(unsafe_code, reason = "required for FFI threadpool detachment")] pub fn detach_threadpool(&self) { unsafe { llama_cpp_bindings_sys::llama_detach_threadpool(self.context.as_ptr()) } } diff --git a/llama-cpp-bindings/src/context/kv_cache_type.rs b/llama-cpp-bindings/src/context/kv_cache_type.rs index 150b5625c..1648ee370 100644 --- a/llama-cpp-bindings/src/context/kv_cache_type.rs +++ b/llama-cpp-bindings/src/context/kv_cache_type.rs @@ -1,51 +1,41 @@ -#[expect( - non_camel_case_types, - reason = "variant names mirror llama.cpp's `enum ggml_type` symbol names verbatim so they can \ - be matched 1:1 against the C ABI without a translation table" -)] -#[expect( - missing_docs, - reason = "each variant denotes a quantisation flavour whose semantics are defined upstream in \ - ggml; restating the upstream spec inline would risk drifting from the source of truth" -)] #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum KvCacheType { Unknown(llama_cpp_bindings_sys::ggml_type), F32, F16, - Q4_0, - Q4_1, - Q5_0, - Q5_1, - Q8_0, - Q8_1, - Q2_K, - Q3_K, - Q4_K, - Q5_K, - Q6_K, - Q8_K, - IQ2_XXS, - IQ2_XS, - IQ3_XXS, - IQ1_S, - IQ4_NL, - IQ3_S, - IQ2_S, - IQ4_XS, + Q40, + Q41, + Q50, + Q51, + Q80, + Q81, + Q2K, + Q3K, + Q4K, + Q5K, + Q6K, + Q8K, + Iq2Xxs, + Iq2Xs, + Iq3Xxs, + Iq1S, + Iq4Nl, + Iq3S, + Iq2S, + Iq4Xs, I8, I16, I32, I64, F64, - IQ1_M, - BF16, - TQ1_0, - TQ2_0, - MXFP4, - NVFP4, - Q1_0, - Q2_0, + Iq1M, + Bf16, + Tq10, + Tq20, + Mxfp4, + Nvfp4, + Q10, + Q20, } impl From for llama_cpp_bindings_sys::ggml_type { @@ -54,39 +44,39 @@ impl From for llama_cpp_bindings_sys::ggml_type { KvCacheType::Unknown(raw) => raw, KvCacheType::F32 => llama_cpp_bindings_sys::GGML_TYPE_F32, KvCacheType::F16 => llama_cpp_bindings_sys::GGML_TYPE_F16, - KvCacheType::Q4_0 => llama_cpp_bindings_sys::GGML_TYPE_Q4_0, - KvCacheType::Q4_1 => llama_cpp_bindings_sys::GGML_TYPE_Q4_1, - KvCacheType::Q5_0 => llama_cpp_bindings_sys::GGML_TYPE_Q5_0, - KvCacheType::Q5_1 => llama_cpp_bindings_sys::GGML_TYPE_Q5_1, - KvCacheType::Q8_0 => llama_cpp_bindings_sys::GGML_TYPE_Q8_0, - KvCacheType::Q8_1 => llama_cpp_bindings_sys::GGML_TYPE_Q8_1, - KvCacheType::Q2_K => llama_cpp_bindings_sys::GGML_TYPE_Q2_K, - KvCacheType::Q3_K => llama_cpp_bindings_sys::GGML_TYPE_Q3_K, - KvCacheType::Q4_K => llama_cpp_bindings_sys::GGML_TYPE_Q4_K, - KvCacheType::Q5_K => llama_cpp_bindings_sys::GGML_TYPE_Q5_K, - KvCacheType::Q6_K => llama_cpp_bindings_sys::GGML_TYPE_Q6_K, - KvCacheType::Q8_K => llama_cpp_bindings_sys::GGML_TYPE_Q8_K, - KvCacheType::IQ2_XXS => llama_cpp_bindings_sys::GGML_TYPE_IQ2_XXS, - KvCacheType::IQ2_XS => llama_cpp_bindings_sys::GGML_TYPE_IQ2_XS, - KvCacheType::IQ3_XXS => llama_cpp_bindings_sys::GGML_TYPE_IQ3_XXS, - KvCacheType::IQ1_S => llama_cpp_bindings_sys::GGML_TYPE_IQ1_S, - KvCacheType::IQ4_NL => llama_cpp_bindings_sys::GGML_TYPE_IQ4_NL, - KvCacheType::IQ3_S => llama_cpp_bindings_sys::GGML_TYPE_IQ3_S, - KvCacheType::IQ2_S => llama_cpp_bindings_sys::GGML_TYPE_IQ2_S, - KvCacheType::IQ4_XS => llama_cpp_bindings_sys::GGML_TYPE_IQ4_XS, + KvCacheType::Q40 => llama_cpp_bindings_sys::GGML_TYPE_Q4_0, + KvCacheType::Q41 => llama_cpp_bindings_sys::GGML_TYPE_Q4_1, + KvCacheType::Q50 => llama_cpp_bindings_sys::GGML_TYPE_Q5_0, + KvCacheType::Q51 => llama_cpp_bindings_sys::GGML_TYPE_Q5_1, + KvCacheType::Q80 => llama_cpp_bindings_sys::GGML_TYPE_Q8_0, + KvCacheType::Q81 => llama_cpp_bindings_sys::GGML_TYPE_Q8_1, + KvCacheType::Q2K => llama_cpp_bindings_sys::GGML_TYPE_Q2_K, + KvCacheType::Q3K => llama_cpp_bindings_sys::GGML_TYPE_Q3_K, + KvCacheType::Q4K => llama_cpp_bindings_sys::GGML_TYPE_Q4_K, + KvCacheType::Q5K => llama_cpp_bindings_sys::GGML_TYPE_Q5_K, + KvCacheType::Q6K => llama_cpp_bindings_sys::GGML_TYPE_Q6_K, + KvCacheType::Q8K => llama_cpp_bindings_sys::GGML_TYPE_Q8_K, + KvCacheType::Iq2Xxs => llama_cpp_bindings_sys::GGML_TYPE_IQ2_XXS, + KvCacheType::Iq2Xs => llama_cpp_bindings_sys::GGML_TYPE_IQ2_XS, + KvCacheType::Iq3Xxs => llama_cpp_bindings_sys::GGML_TYPE_IQ3_XXS, + KvCacheType::Iq1S => llama_cpp_bindings_sys::GGML_TYPE_IQ1_S, + KvCacheType::Iq4Nl => llama_cpp_bindings_sys::GGML_TYPE_IQ4_NL, + KvCacheType::Iq3S => llama_cpp_bindings_sys::GGML_TYPE_IQ3_S, + KvCacheType::Iq2S => llama_cpp_bindings_sys::GGML_TYPE_IQ2_S, + KvCacheType::Iq4Xs => llama_cpp_bindings_sys::GGML_TYPE_IQ4_XS, KvCacheType::I8 => llama_cpp_bindings_sys::GGML_TYPE_I8, KvCacheType::I16 => llama_cpp_bindings_sys::GGML_TYPE_I16, KvCacheType::I32 => llama_cpp_bindings_sys::GGML_TYPE_I32, KvCacheType::I64 => llama_cpp_bindings_sys::GGML_TYPE_I64, KvCacheType::F64 => llama_cpp_bindings_sys::GGML_TYPE_F64, - KvCacheType::IQ1_M => llama_cpp_bindings_sys::GGML_TYPE_IQ1_M, - KvCacheType::BF16 => llama_cpp_bindings_sys::GGML_TYPE_BF16, - KvCacheType::TQ1_0 => llama_cpp_bindings_sys::GGML_TYPE_TQ1_0, - KvCacheType::TQ2_0 => llama_cpp_bindings_sys::GGML_TYPE_TQ2_0, - KvCacheType::MXFP4 => llama_cpp_bindings_sys::GGML_TYPE_MXFP4, - KvCacheType::NVFP4 => llama_cpp_bindings_sys::GGML_TYPE_NVFP4, - KvCacheType::Q1_0 => llama_cpp_bindings_sys::GGML_TYPE_Q1_0, - KvCacheType::Q2_0 => llama_cpp_bindings_sys::GGML_TYPE_Q2_0, + KvCacheType::Iq1M => llama_cpp_bindings_sys::GGML_TYPE_IQ1_M, + KvCacheType::Bf16 => llama_cpp_bindings_sys::GGML_TYPE_BF16, + KvCacheType::Tq10 => llama_cpp_bindings_sys::GGML_TYPE_TQ1_0, + KvCacheType::Tq20 => llama_cpp_bindings_sys::GGML_TYPE_TQ2_0, + KvCacheType::Mxfp4 => llama_cpp_bindings_sys::GGML_TYPE_MXFP4, + KvCacheType::Nvfp4 => llama_cpp_bindings_sys::GGML_TYPE_NVFP4, + KvCacheType::Q10 => llama_cpp_bindings_sys::GGML_TYPE_Q1_0, + KvCacheType::Q20 => llama_cpp_bindings_sys::GGML_TYPE_Q2_0, } } } @@ -94,41 +84,41 @@ impl From for llama_cpp_bindings_sys::ggml_type { impl From for KvCacheType { fn from(value: llama_cpp_bindings_sys::ggml_type) -> Self { match value { - x if x == llama_cpp_bindings_sys::GGML_TYPE_F32 => Self::F32, - x if x == llama_cpp_bindings_sys::GGML_TYPE_F16 => Self::F16, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q4_0 => Self::Q4_0, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q4_1 => Self::Q4_1, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q5_0 => Self::Q5_0, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q5_1 => Self::Q5_1, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q8_0 => Self::Q8_0, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q8_1 => Self::Q8_1, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q2_K => Self::Q2_K, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q3_K => Self::Q3_K, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q4_K => Self::Q4_K, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q5_K => Self::Q5_K, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q6_K => Self::Q6_K, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q8_K => Self::Q8_K, - x if x == llama_cpp_bindings_sys::GGML_TYPE_IQ2_XXS => Self::IQ2_XXS, - x if x == llama_cpp_bindings_sys::GGML_TYPE_IQ2_XS => Self::IQ2_XS, - x if x == llama_cpp_bindings_sys::GGML_TYPE_IQ3_XXS => Self::IQ3_XXS, - x if x == llama_cpp_bindings_sys::GGML_TYPE_IQ1_S => Self::IQ1_S, - x if x == llama_cpp_bindings_sys::GGML_TYPE_IQ4_NL => Self::IQ4_NL, - x if x == llama_cpp_bindings_sys::GGML_TYPE_IQ3_S => Self::IQ3_S, - x if x == llama_cpp_bindings_sys::GGML_TYPE_IQ2_S => Self::IQ2_S, - x if x == llama_cpp_bindings_sys::GGML_TYPE_IQ4_XS => Self::IQ4_XS, - x if x == llama_cpp_bindings_sys::GGML_TYPE_I8 => Self::I8, - x if x == llama_cpp_bindings_sys::GGML_TYPE_I16 => Self::I16, - x if x == llama_cpp_bindings_sys::GGML_TYPE_I32 => Self::I32, - x if x == llama_cpp_bindings_sys::GGML_TYPE_I64 => Self::I64, - x if x == llama_cpp_bindings_sys::GGML_TYPE_F64 => Self::F64, - x if x == llama_cpp_bindings_sys::GGML_TYPE_IQ1_M => Self::IQ1_M, - x if x == llama_cpp_bindings_sys::GGML_TYPE_BF16 => Self::BF16, - x if x == llama_cpp_bindings_sys::GGML_TYPE_TQ1_0 => Self::TQ1_0, - x if x == llama_cpp_bindings_sys::GGML_TYPE_TQ2_0 => Self::TQ2_0, - x if x == llama_cpp_bindings_sys::GGML_TYPE_MXFP4 => Self::MXFP4, - x if x == llama_cpp_bindings_sys::GGML_TYPE_NVFP4 => Self::NVFP4, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q1_0 => Self::Q1_0, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q2_0 => Self::Q2_0, + llama_cpp_bindings_sys::GGML_TYPE_F32 => Self::F32, + llama_cpp_bindings_sys::GGML_TYPE_F16 => Self::F16, + llama_cpp_bindings_sys::GGML_TYPE_Q4_0 => Self::Q40, + llama_cpp_bindings_sys::GGML_TYPE_Q4_1 => Self::Q41, + llama_cpp_bindings_sys::GGML_TYPE_Q5_0 => Self::Q50, + llama_cpp_bindings_sys::GGML_TYPE_Q5_1 => Self::Q51, + llama_cpp_bindings_sys::GGML_TYPE_Q8_0 => Self::Q80, + llama_cpp_bindings_sys::GGML_TYPE_Q8_1 => Self::Q81, + llama_cpp_bindings_sys::GGML_TYPE_Q2_K => Self::Q2K, + llama_cpp_bindings_sys::GGML_TYPE_Q3_K => Self::Q3K, + llama_cpp_bindings_sys::GGML_TYPE_Q4_K => Self::Q4K, + llama_cpp_bindings_sys::GGML_TYPE_Q5_K => Self::Q5K, + llama_cpp_bindings_sys::GGML_TYPE_Q6_K => Self::Q6K, + llama_cpp_bindings_sys::GGML_TYPE_Q8_K => Self::Q8K, + llama_cpp_bindings_sys::GGML_TYPE_IQ2_XXS => Self::Iq2Xxs, + llama_cpp_bindings_sys::GGML_TYPE_IQ2_XS => Self::Iq2Xs, + llama_cpp_bindings_sys::GGML_TYPE_IQ3_XXS => Self::Iq3Xxs, + llama_cpp_bindings_sys::GGML_TYPE_IQ1_S => Self::Iq1S, + llama_cpp_bindings_sys::GGML_TYPE_IQ4_NL => Self::Iq4Nl, + llama_cpp_bindings_sys::GGML_TYPE_IQ3_S => Self::Iq3S, + llama_cpp_bindings_sys::GGML_TYPE_IQ2_S => Self::Iq2S, + llama_cpp_bindings_sys::GGML_TYPE_IQ4_XS => Self::Iq4Xs, + llama_cpp_bindings_sys::GGML_TYPE_I8 => Self::I8, + llama_cpp_bindings_sys::GGML_TYPE_I16 => Self::I16, + llama_cpp_bindings_sys::GGML_TYPE_I32 => Self::I32, + llama_cpp_bindings_sys::GGML_TYPE_I64 => Self::I64, + llama_cpp_bindings_sys::GGML_TYPE_F64 => Self::F64, + llama_cpp_bindings_sys::GGML_TYPE_IQ1_M => Self::Iq1M, + llama_cpp_bindings_sys::GGML_TYPE_BF16 => Self::Bf16, + llama_cpp_bindings_sys::GGML_TYPE_TQ1_0 => Self::Tq10, + llama_cpp_bindings_sys::GGML_TYPE_TQ2_0 => Self::Tq20, + llama_cpp_bindings_sys::GGML_TYPE_MXFP4 => Self::Mxfp4, + llama_cpp_bindings_sys::GGML_TYPE_NVFP4 => Self::Nvfp4, + llama_cpp_bindings_sys::GGML_TYPE_Q1_0 => Self::Q10, + llama_cpp_bindings_sys::GGML_TYPE_Q2_0 => Self::Q20, _ => Self::Unknown(value), } } @@ -181,39 +171,39 @@ mod tests { let all_variants = [ KvCacheType::F32, KvCacheType::F16, - KvCacheType::Q4_0, - KvCacheType::Q4_1, - KvCacheType::Q5_0, - KvCacheType::Q5_1, - KvCacheType::Q8_0, - KvCacheType::Q8_1, - KvCacheType::Q2_K, - KvCacheType::Q3_K, - KvCacheType::Q4_K, - KvCacheType::Q5_K, - KvCacheType::Q6_K, - KvCacheType::Q8_K, - KvCacheType::IQ2_XXS, - KvCacheType::IQ2_XS, - KvCacheType::IQ3_XXS, - KvCacheType::IQ1_S, - KvCacheType::IQ4_NL, - KvCacheType::IQ3_S, - KvCacheType::IQ2_S, - KvCacheType::IQ4_XS, + KvCacheType::Q40, + KvCacheType::Q41, + KvCacheType::Q50, + KvCacheType::Q51, + KvCacheType::Q80, + KvCacheType::Q81, + KvCacheType::Q2K, + KvCacheType::Q3K, + KvCacheType::Q4K, + KvCacheType::Q5K, + KvCacheType::Q6K, + KvCacheType::Q8K, + KvCacheType::Iq2Xxs, + KvCacheType::Iq2Xs, + KvCacheType::Iq3Xxs, + KvCacheType::Iq1S, + KvCacheType::Iq4Nl, + KvCacheType::Iq3S, + KvCacheType::Iq2S, + KvCacheType::Iq4Xs, KvCacheType::I8, KvCacheType::I16, KvCacheType::I32, KvCacheType::I64, KvCacheType::F64, - KvCacheType::IQ1_M, - KvCacheType::BF16, - KvCacheType::TQ1_0, - KvCacheType::TQ2_0, - KvCacheType::MXFP4, - KvCacheType::NVFP4, - KvCacheType::Q1_0, - KvCacheType::Q2_0, + KvCacheType::Iq1M, + KvCacheType::Bf16, + KvCacheType::Tq10, + KvCacheType::Tq20, + KvCacheType::Mxfp4, + KvCacheType::Nvfp4, + KvCacheType::Q10, + KvCacheType::Q20, ]; for variant in all_variants { diff --git a/llama-cpp-bindings/src/context/params.rs b/llama-cpp-bindings/src/context/params.rs index 9ec231939..4ad588309 100644 --- a/llama-cpp-bindings/src/context/params.rs +++ b/llama-cpp-bindings/src/context/params.rs @@ -7,12 +7,6 @@ pub use crate::context::llama_pooling_type::LlamaPoolingType; pub use crate::context::rope_scaling_type::RopeScalingType; #[derive(Debug, Clone, Copy)] -#[expect( - missing_docs, - reason = "field meanings mirror llama.cpp's `llama_context_params` C struct; restating each \ - one inline would risk drift from the upstream spec — the doc-comment on the struct \ - points at the canonical reference" -)] pub struct LlamaContextParams { pub context_params: llama_cpp_bindings_sys::llama_context_params, } @@ -519,16 +513,16 @@ mod tests { #[test] fn with_type_k_sets_value() { - let params = super::LlamaContextParams::default().with_type_k(KvCacheType::Q4_0); + let params = super::LlamaContextParams::default().with_type_k(KvCacheType::Q40); - assert_eq!(params.type_k(), KvCacheType::Q4_0); + assert_eq!(params.type_k(), KvCacheType::Q40); } #[test] fn with_type_v_sets_value() { - let params = super::LlamaContextParams::default().with_type_v(KvCacheType::Q4_1); + let params = super::LlamaContextParams::default().with_type_v(KvCacheType::Q41); - assert_eq!(params.type_v(), KvCacheType::Q4_1); + assert_eq!(params.type_v(), KvCacheType::Q41); } #[test] diff --git a/llama-cpp-bindings/src/cpp_text_view.rs b/llama-cpp-bindings/src/cpp_text_view.rs new file mode 100644 index 000000000..92088fee1 --- /dev/null +++ b/llama-cpp-bindings/src/cpp_text_view.rs @@ -0,0 +1,20 @@ +use std::ffi::c_char; +use std::slice; +use std::str; +use std::str::Utf8Error; + +/// Borrows the UTF-8 text of a C++ string llama.cpp exposes as a pointer and a length. +/// +/// # Safety +/// +/// `text` must point to `length` readable bytes that stay alive and unchanged for `'text`. +/// +/// # Errors +/// +/// Returns [`Utf8Error`] when the bytes are not UTF-8. +pub const unsafe fn cpp_text_view<'text>( + text: *const c_char, + length: usize, +) -> Result<&'text str, Utf8Error> { + str::from_utf8(unsafe { slice::from_raw_parts(text.cast::(), length) }) +} diff --git a/llama-cpp-bindings/src/error.rs b/llama-cpp-bindings/src/error.rs index 80c851c0d..374b80e73 100644 --- a/llama-cpp-bindings/src/error.rs +++ b/llama-cpp-bindings/src/error.rs @@ -1,7 +1,8 @@ -pub mod apply_chat_template_error; pub mod batch_transfer_error; pub mod bracketed_args_failure; +pub mod chat_session_error; pub mod chat_template_error; +pub mod chat_templates_error; pub mod clear_kv_cache_seq_error; pub mod copy_kv_cache_seq_error; pub mod decode_error; @@ -11,6 +12,7 @@ pub mod eval_multimodal_chunks_error; pub mod fit_error; pub mod grammar_error; pub mod grammar_runtime_error; +pub mod ingest_error; pub mod json_object_failure; pub mod json_schema_to_grammar_error; pub mod key_value_xml_tags_failure; @@ -23,7 +25,6 @@ pub mod llama_lora_adapter_init_error; pub mod llama_lora_adapters_error; pub mod llama_model_load_error; pub mod logits_error; -pub mod marker_detection_error; pub mod meta_val_error; pub mod model_params_error; pub mod new_llama_chat_message_error; @@ -45,9 +46,10 @@ pub mod xml_function_tags_failure; pub use llama_cpp_ffi_status::FfiContractError; pub use llama_cpp_ffi_status::FfiStatusError; -pub use apply_chat_template_error::ApplyChatTemplateError; pub use bracketed_args_failure::BracketedArgsFailure; +pub use chat_session_error::ChatSessionError; pub use chat_template_error::ChatTemplateError; +pub use chat_templates_error::ChatTemplatesError; pub use clear_kv_cache_seq_error::ClearKvCacheSeqError; pub use copy_kv_cache_seq_error::CopyKvCacheSeqError; pub use decode_error::DecodeError; @@ -57,6 +59,7 @@ pub use eval_multimodal_chunks_error::EvalMultimodalChunksError; pub use fit_error::FitError; pub use grammar_error::GrammarError; pub use grammar_runtime_error::GrammarRuntimeError; +pub use ingest_error::IngestError; pub use json_object_failure::JsonObjectFailure; pub use json_schema_to_grammar_error::JsonSchemaToGrammarError; pub use key_value_xml_tags_failure::KeyValueXmlTagsFailure; @@ -69,7 +72,6 @@ pub use llama_lora_adapter_init_error::LlamaLoraAdapterInitError; pub use llama_lora_adapters_error::LlamaLoraAdaptersError; pub use llama_model_load_error::LlamaModelLoadError; pub use logits_error::LogitsError; -pub use marker_detection_error::MarkerDetectionError; pub use meta_val_error::MetaValError; pub use model_params_error::ModelParamsError; pub use new_llama_chat_message_error::NewLlamaChatMessageError; diff --git a/llama-cpp-bindings/src/error/apply_chat_template_error.rs b/llama-cpp-bindings/src/error/apply_chat_template_error.rs deleted file mode 100644 index 12a53c44e..000000000 --- a/llama-cpp-bindings/src/error/apply_chat_template_error.rs +++ /dev/null @@ -1,17 +0,0 @@ -#[derive(Debug, PartialEq, Eq, thiserror::Error)] -pub enum ApplyChatTemplateError { - #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), - #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), - #[error("the model has no vocab")] - NoVocab, - #[error("the model's chat template rendered an empty prompt or could not be rendered")] - TemplateApplicationFailed, - #[error("not enough memory to render the chat template")] - NotEnoughMemory, - #[error("the llama.cpp chat-template renderer ran out of memory")] - LlamaCppOutOfMemory, - #[error("{message}")] - Reported { message: String }, -} diff --git a/llama-cpp-bindings/src/error/chat_session_error.rs b/llama-cpp-bindings/src/error/chat_session_error.rs new file mode 100644 index 000000000..f8355a8c4 --- /dev/null +++ b/llama-cpp-bindings/src/error/chat_session_error.rs @@ -0,0 +1,45 @@ +use std::ffi::NulError; +use std::str::Utf8Error; + +use crate::error::chat_templates_error::ChatTemplatesError; +use crate::error::grammar_error::GrammarError; +use crate::error::sampler_accept_error::SamplerAcceptError; +use crate::error::string_to_token_error::StringToTokenError; + +#[derive(Debug, Eq, PartialEq, thiserror::Error)] +pub enum ChatSessionError { + #[error(transparent)] + FfiStatus(#[from] crate::FfiStatusError), + #[error(transparent)] + FfiContract(#[from] crate::FfiContractError), + #[error("the model's chat templates are unavailable: {0}")] + Templates(#[from] ChatTemplatesError), + #[error("a chat message is not valid UTF-8")] + MessageNotUtf8(#[source] Utf8Error), + #[error("the chat request contains an interior NUL byte")] + RequestContainsNul(#[source] NulError), + #[error("the messages are not valid JSON: {message}")] + MessagesNotJson { message: String }, + #[error("the messages are not valid chat messages: {message}")] + MessagesInvalid { message: String }, + #[error("the tools are not valid JSON: {message}")] + ToolsNotJson { message: String }, + #[error("the tools are not valid tool definitions: {message}")] + ToolsInvalid { message: String }, + #[error("the chat template cannot serve the request: {message}")] + TemplateCannotServeRequest { message: String }, + #[error("text the chat session holds is not valid UTF-8")] + SessionTextNotUtf8(#[source] Utf8Error), + #[error("the rendered prompt could not be tokenized: {0}")] + PromptTokenization(#[from] StringToTokenError), + #[error("the request's grammar sampler could not be built: {0}")] + Grammar(#[from] GrammarError), + #[error("the grammar sampler refused a generation-prompt token: {0}")] + GrammarPrefill(#[from] SamplerAcceptError), + #[error("not enough memory")] + NotEnoughMemory, + #[error("the llama.cpp library ran out of memory")] + LlamaCppOutOfMemory, + #[error("{message}")] + Reported { message: String }, +} diff --git a/llama-cpp-bindings/src/error/chat_templates_error.rs b/llama-cpp-bindings/src/error/chat_templates_error.rs new file mode 100644 index 000000000..eba331ad1 --- /dev/null +++ b/llama-cpp-bindings/src/error/chat_templates_error.rs @@ -0,0 +1,15 @@ +#[derive(Debug, Eq, PartialEq, thiserror::Error)] +pub enum ChatTemplatesError { + #[error(transparent)] + FfiStatus(#[from] crate::FfiStatusError), + #[error(transparent)] + FfiContract(#[from] crate::FfiContractError), + #[error("the model has no chat template")] + ModelHasNoChatTemplate, + #[error("not enough memory")] + NotEnoughMemory, + #[error("the llama.cpp library ran out of memory")] + LlamaCppOutOfMemory, + #[error("{message}")] + Reported { message: String }, +} diff --git a/llama-cpp-bindings/src/error/ingest_error.rs b/llama-cpp-bindings/src/error/ingest_error.rs new file mode 100644 index 000000000..b380f2bd9 --- /dev/null +++ b/llama-cpp-bindings/src/error/ingest_error.rs @@ -0,0 +1,10 @@ +use crate::error::parse_chat_message_error::ParseChatMessageError; +use crate::error::token_to_string_error::TokenToStringError; + +#[derive(Debug, Eq, PartialEq, thiserror::Error)] +pub enum IngestError { + #[error("the generated token could not be detokenized: {0}")] + Detokenize(#[from] TokenToStringError), + #[error("the chat template's parser could not classify the answer: {0}")] + Classify(#[from] ParseChatMessageError), +} diff --git a/llama-cpp-bindings/src/error/marker_detection_error.rs b/llama-cpp-bindings/src/error/marker_detection_error.rs deleted file mode 100644 index c3740617e..000000000 --- a/llama-cpp-bindings/src/error/marker_detection_error.rs +++ /dev/null @@ -1,47 +0,0 @@ -use std::str::Utf8Error; -use std::string::FromUtf8Error; - -use crate::error::chat_template_error::ChatTemplateError; -use crate::error::string_to_token_error::StringToTokenError; -use crate::token::LlamaToken; - -#[derive(Debug, PartialEq, Eq, thiserror::Error)] -pub enum MarkerDetectionError { - #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), - #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), - #[error("ffi returned non-utf8 marker bytes: {0}")] - MarkerUtf8Error(#[from] FromUtf8Error), - #[error("not enough memory")] - NotEnoughMemory, - #[error("{operation} could not run because the model has no chat template")] - ModelHasNoChatTemplate { operation: &'static str }, - #[error("{operation} could not run because the model has no vocab")] - ModelHasNoVocab { operation: &'static str }, - #[error("the llama.cpp library ran out of memory")] - LlamaCppOutOfMemory, - #[error("reasoning-marker detection failed: {message}")] - ReasoningMarkerDetectionFailed { message: String }, - #[error("tool-call haystack computation failed: {message}")] - ToolCallHaystackComputationFailed { message: String }, - #[error("tool-call synthetic-render diagnosis failed: {message}")] - ToolCallSyntheticRenderDiagnosisFailed { message: String }, - #[error("the reasoning-markers destructor threw: {message}")] - ReasoningMarkersFreeFailed { message: String }, - #[error("a detected marker string could not be tokenised: {0}")] - MarkerTokenizationFailed(#[from] StringToTokenError), - #[error("marker detection produced an empty token sequence")] - EmptyMarker, - #[error("marker token sequence {tokens:?} opens more than one section")] - AmbiguousMarkerOpeners { tokens: Vec }, - #[error("the chat template is not valid UTF-8: {0}")] - ToolCallTemplateNotUtf8(#[from] Utf8Error), - #[error("the chat template could not be retrieved for tool-call marker detection: {0}")] - ChatTemplateUnavailable(#[source] ChatTemplateError), - #[error("{operation} rejected the Rust-owned argument {argument}")] - WrapperRejectedArgument { - operation: &'static str, - argument: &'static str, - }, -} diff --git a/llama-cpp-bindings/src/error/parse_chat_message_error.rs b/llama-cpp-bindings/src/error/parse_chat_message_error.rs index 3287d137d..073811cc9 100644 --- a/llama-cpp-bindings/src/error/parse_chat_message_error.rs +++ b/llama-cpp-bindings/src/error/parse_chat_message_error.rs @@ -1,6 +1,4 @@ -use std::string::FromUtf8Error; - -use crate::error::marker_detection_error::MarkerDetectionError; +use std::str::Utf8Error; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum ParseChatMessageError { @@ -8,38 +6,14 @@ pub enum ParseChatMessageError { FfiStatus(#[from] crate::FfiStatusError), #[error(transparent)] FfiContract(#[from] crate::FfiContractError), - #[error("model has no chat template")] - NoChatTemplate, - #[error("model has no vocab")] - NoVocab, #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] LlamaCppOutOfMemory, - #[error("the chat parser could not be constructed: {message}")] - ParserCreationFailed { message: String }, - #[error("the chat tools contain an interior NUL byte")] - ToolsContainNulByte(#[source] std::ffi::NulError), - #[error("the chat tools are not a JSON array")] - ToolsNotAnArray, - #[error("the chat parser could not be built for the given tools: {message}")] - ToolsParserBuildFailed { message: String }, - #[error("the chat parser did not recognize the message: {message}")] - MessageUnrecognized { message: String }, - #[error("the chat parser destructor threw: {message}")] + #[error("the parsed chat destructor threw: {message}")] DestructorFailed { message: String }, - #[error("tool-call id index {index} out of bounds")] - ToolCallIdIndexOutOfBounds { index: usize }, - #[error("tool-call name index {index} out of bounds")] - ToolCallNameIndexOutOfBounds { index: usize }, - #[error("tool-call arguments index {index} out of bounds")] - ToolCallArgumentsIndexOutOfBounds { index: usize }, - #[error("ffi returned non-utf8 string: {0}")] - StringUtf8Error(#[from] FromUtf8Error), - #[error("the message to parse contains an interior NUL byte")] - InputContainsNulByte(#[source] std::ffi::NulError), - #[error("reasoning-marker detection failed: {0}")] - MarkerDetection(#[from] MarkerDetectionError), + #[error("text llama.cpp parsed out of the answer is not valid UTF-8: {0}")] + ParsedTextNotUtf8(#[source] Utf8Error), #[error("{message}")] Reported { message: String }, } diff --git a/llama-cpp-bindings/src/error/sample_error.rs b/llama-cpp-bindings/src/error/sample_error.rs index 21c554d91..8fe7da983 100644 --- a/llama-cpp-bindings/src/error/sample_error.rs +++ b/llama-cpp-bindings/src/error/sample_error.rs @@ -1,6 +1,6 @@ +use crate::error::ingest_error::IngestError; use crate::error::logits_error::LogitsError; use crate::error::sampler_apply_error::SamplerApplyError; -use crate::error::token_to_string_error::TokenToStringError; #[derive(Debug, thiserror::Error, PartialEq, Eq)] pub enum SampleError { @@ -16,8 +16,8 @@ pub enum SampleError { LlamaCppOutOfMemory, #[error("applying the sampler to the token data array failed: {0}")] SamplerApply(#[from] SamplerApplyError), - #[error("token detokenization failed during classification: {0}")] - Detokenize(#[from] TokenToStringError), + #[error("classifying the sampled token failed: {0}")] + Ingest(#[from] IngestError), #[error("the grammar sampler callback failed during sampling: {message}")] GrammarCallbackFailed { message: String }, #[error("{message}")] diff --git a/llama-cpp-bindings/src/extract_tool_call_markers_from_haystack.rs b/llama-cpp-bindings/src/extract_tool_call_markers_from_haystack.rs deleted file mode 100644 index bbbbdb4e6..000000000 --- a/llama-cpp-bindings/src/extract_tool_call_markers_from_haystack.rs +++ /dev/null @@ -1,143 +0,0 @@ -use crate::tool_call_marker_pair::ToolCallMarkerPair; - -#[must_use] -pub fn extract_tool_call_markers_from_haystack(haystack: &str) -> Option { - if haystack.is_empty() { - return None; - } - - let json_start = haystack.find('{')?; - let json_end = haystack.rfind('}')?; - if json_end < json_start { - return None; - } - - let json_slice = &haystack[json_start..=json_end]; - serde_json::from_str::(json_slice).ok()?; - - let open = haystack[..json_start].trim().to_owned(); - let close = haystack[json_end + 1..].trim().to_owned(); - - if open.is_empty() || close.is_empty() { - return None; - } - - Some(ToolCallMarkerPair { open, close }) -} - -#[cfg(test)] -mod tests { - use super::ToolCallMarkerPair; - use super::extract_tool_call_markers_from_haystack; - - #[test] - fn extracts_open_and_close_around_a_simple_json_payload() { - let pair = extract_tool_call_markers_from_haystack( - "{\"name\":\"x\",\"arguments\":{}}", - ); - - assert_eq!( - pair, - Some(ToolCallMarkerPair { - open: "".to_owned(), - close: "".to_owned(), - }), - ); - } - - #[test] - fn trims_surrounding_whitespace_from_each_marker() { - let pair = extract_tool_call_markers_from_haystack( - " \n {\"k\": 1}\n ", - ); - - assert_eq!( - pair, - Some(ToolCallMarkerPair { - open: "".to_owned(), - close: "".to_owned(), - }), - ); - } - - #[test] - fn returns_none_when_haystack_is_empty() { - assert_eq!(extract_tool_call_markers_from_haystack(""), None); - } - - #[test] - fn returns_none_when_haystack_has_no_open_brace() { - assert_eq!( - extract_tool_call_markers_from_haystack("plain assistant text"), - None - ); - } - - #[test] - fn returns_none_when_haystack_has_open_brace_but_no_close() { - assert_eq!( - extract_tool_call_markers_from_haystack("{ unclosed"), - None - ); - } - - #[test] - fn returns_none_when_close_brace_precedes_open_brace() { - assert_eq!( - extract_tool_call_markers_from_haystack("}{"), - None - ); - } - - #[test] - fn returns_none_when_brace_payload_is_not_valid_json() { - assert_eq!( - extract_tool_call_markers_from_haystack("{not valid json}"), - None - ); - } - - #[test] - fn returns_none_when_open_marker_resolves_to_empty_after_trim() { - assert_eq!( - extract_tool_call_markers_from_haystack(" {\"x\":1}"), - None - ); - } - - #[test] - fn returns_none_when_close_marker_resolves_to_empty_after_trim() { - assert_eq!( - extract_tool_call_markers_from_haystack("{\"x\":1} "), - None - ); - } - - #[test] - fn extracts_around_an_object_that_contains_nested_braces() { - let pair = extract_tool_call_markers_from_haystack( - "{\"args\":{\"k\":[1,2,{\"deep\":true}]}}", - ); - - assert_eq!( - pair, - Some(ToolCallMarkerPair { - open: "".to_owned(), - close: "".to_owned(), - }), - ); - } - - #[test] - fn extracts_when_open_marker_contains_multibyte_utf8() { - let pair = extract_tool_call_markers_from_haystack("<|tool→call|>{\"k\":1}<|/tool→call|>"); - - assert_eq!( - pair, - Some(ToolCallMarkerPair { - open: "<|tool→call|>".to_owned(), - close: "<|/tool→call|>".to_owned(), - }), - ); - } -} diff --git a/llama-cpp-bindings/src/lib.rs b/llama-cpp-bindings/src/lib.rs index c1bb528c1..040c10f8d 100644 --- a/llama-cpp-bindings/src/lib.rs +++ b/llama-cpp-bindings/src/lib.rs @@ -3,17 +3,33 @@ deny(clippy::unwrap_used, clippy::expect_used, clippy::panic) )] +pub mod bare_json_tool_call_probe; pub mod bare_json_tool_calls; pub mod batch_add_error; +pub mod chat_assistant_prefill; +pub mod chat_assistant_turn; +pub mod chat_classification; +pub mod chat_grammar_trigger; pub mod chat_message_parse_outcome; -pub mod chat_message_parser; +pub mod chat_output; +pub mod chat_output_completeness; +pub mod chat_output_token; +pub mod chat_piece_role; +pub mod chat_prompt_part; +pub mod chat_session; +pub mod chat_session_options; +pub mod chat_span; +pub mod chat_span_kind; +pub mod chat_spans; pub mod chat_template_tool_calls; +pub mod chat_templates; +pub mod chat_tool_choice; pub mod classified_sample; pub mod context; +pub mod cpp_text_view; pub mod dry_sequence_breakers; pub mod error; pub mod eval_multimodal_chunks_params; -pub mod extract_tool_call_markers_from_haystack; pub mod generation_progress; pub mod ggml_time_us; pub mod gguf_context; @@ -44,16 +60,15 @@ pub mod load_backends_error; pub mod load_backends_from_path; pub mod log_options; pub mod log_record; -pub mod marker_role; -pub mod marker_role_candidate; pub mod mask_outcome; pub mod max_devices; pub mod mlock_supported; pub mod mmap_supported; pub mod model; pub mod mtmd; +pub mod parsed_chat; pub mod raw_chat_message; -pub mod resolved_tool_call_markers; +pub mod result_after_release; pub mod sampled_token; pub mod sampled_token_classifier; pub mod sampled_token_section; @@ -61,29 +76,35 @@ pub mod sampling; pub mod sanitized_grammar; pub mod send_logs_to_log; pub mod streaming_json_probe; -pub mod streaming_marker; -pub mod streaming_markers; -pub mod synthetic_tool_call_renders; pub mod timing; pub mod token; pub mod token_piece; pub mod tool_call_format; -pub mod tool_call_marker_pair; pub use error::{ - ApplyChatTemplateError, ChatTemplateError, ClearKvCacheSeqError, CopyKvCacheSeqError, - DecodeError, EmbeddingsError, EncodeError, EvalMultimodalChunksError, FfiContractError, - FfiStatusError, GrammarError, JsonSchemaToGrammarError, KvCacheSeqAddError, KvCacheSeqDivError, - KvCacheSeqPosMaxError, LlamaContextLoadError, LlamaCppError, LlamaLoraAdapterInitError, - LlamaLoraAdaptersError, LlamaModelLoadError, LogitsError, MarkerDetectionError, MetaValError, - ModelParamsError, NewLlamaChatMessageError, ParseChatMessageError, Result, SampleError, - SamplerAcceptError, SamplingError, StringToTokenError, TokenAttrError, TokenOutsideVocabulary, - TokenSamplingError, TokenToStringError, + ChatSessionError, ChatTemplateError, ChatTemplatesError, ClearKvCacheSeqError, + CopyKvCacheSeqError, DecodeError, EmbeddingsError, EncodeError, EvalMultimodalChunksError, + FfiContractError, FfiStatusError, GrammarError, IngestError, JsonSchemaToGrammarError, + KvCacheSeqAddError, KvCacheSeqDivError, KvCacheSeqPosMaxError, LlamaContextLoadError, + LlamaCppError, LlamaLoraAdapterInitError, LlamaLoraAdaptersError, LlamaModelLoadError, + LogitsError, MetaValError, ModelParamsError, NewLlamaChatMessageError, ParseChatMessageError, + Result, SampleError, SamplerAcceptError, SamplingError, StringToTokenError, TokenAttrError, + TokenOutsideVocabulary, TokenSamplingError, TokenToStringError, }; pub use bare_json_tool_calls::BareJsonToolCalls; +pub use chat_assistant_prefill::ChatAssistantPrefill; +pub use chat_assistant_turn::ChatAssistantTurn; +pub use chat_classification::ChatClassification; +pub use chat_grammar_trigger::ChatGrammarTrigger; pub use chat_message_parse_outcome::ChatMessageParseOutcome; -pub use chat_message_parser::ChatMessageParser; +pub use chat_output::ChatOutput; +pub use chat_output_completeness::ChatOutputCompleteness; +pub use chat_prompt_part::ChatPromptPart; +pub use chat_session::ChatSession; +pub use chat_session_options::ChatSessionOptions; +pub use chat_templates::ChatTemplates; +pub use chat_tool_choice::ChatToolChoice; pub use classified_sample::ClassifiedSample; pub use eval_multimodal_chunks_params::EvalMultimodalChunksParams; pub use generation_progress::GenerationProgress; @@ -91,19 +112,14 @@ pub use llama_backend_device::LlamaBackendDevice; pub use llama_backend_device_type::LlamaBackendDeviceType; pub use llama_cpp_bindings_types::{ BracketedJsonShape, KeyValueXmlTagsShape, PairedQuoteShape, ParsedChatMessage, ParsedToolCall, - ParsedToolCallId, ReasoningMarkers, TokenUsage, TokenUsageError, ToolCallArgsShape, - ToolCallArguments, ToolCallMarkers, ToolCallValueQuote, XmlTagsShape, + ParsedToolCallId, TokenUsage, TokenUsageError, ToolCallArgsShape, ToolCallArguments, + ToolCallMarkers, ToolCallValueQuote, XmlTagsShape, }; pub use llama_cpp_gbnf::gbnf_validation_error::GbnfValidationError; -pub use marker_role::MarkerRole; -pub use marker_role_candidate::MarkerRoleCandidate; pub use raw_chat_message::RawChatMessage; pub use sampled_token::SampledToken; pub use sampled_token_classifier::SampledTokenClassifier; pub use sampled_token_section::SampledTokenSection; -pub use streaming_marker::StreamingMarker; -pub use streaming_markers::StreamingMarkers; -pub use synthetic_tool_call_renders::SyntheticToolCallRenders; pub use token_piece::TokenPiece; pub use ggml_time_us::ggml_time_us; diff --git a/llama-cpp-bindings/src/marker_role.rs b/llama-cpp-bindings/src/marker_role.rs deleted file mode 100644 index 665a4fba5..000000000 --- a/llama-cpp-bindings/src/marker_role.rs +++ /dev/null @@ -1,79 +0,0 @@ -use crate::sampled_token_section::SampledTokenSection; - -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -/// A semantic transition performed when a streaming marker is consumed. -pub enum MarkerRole { - ReasoningOpen, - ReasoningClose, - ToolCallOpen, - ToolCallClose, -} - -impl MarkerRole { - #[must_use] - pub const fn opened_section(self) -> Option { - match self { - Self::ReasoningOpen => Some(SampledTokenSection::Reasoning), - Self::ToolCallOpen => Some(SampledTokenSection::ToolCall), - Self::ReasoningClose | Self::ToolCallClose => None, - } - } - - #[must_use] - pub const fn closed_section(self) -> Option { - match self { - Self::ReasoningClose => Some(SampledTokenSection::Reasoning), - Self::ToolCallClose => Some(SampledTokenSection::ToolCall), - Self::ReasoningOpen | Self::ToolCallOpen => None, - } - } -} - -#[cfg(test)] -mod tests { - use super::MarkerRole; - use crate::sampled_token_section::SampledTokenSection; - - struct RoleSection { - role: MarkerRole, - section: SampledTokenSection, - } - - #[test] - fn an_opening_role_opens_its_section_and_closes_nothing() { - let cases = [ - RoleSection { - role: MarkerRole::ReasoningOpen, - section: SampledTokenSection::Reasoning, - }, - RoleSection { - role: MarkerRole::ToolCallOpen, - section: SampledTokenSection::ToolCall, - }, - ]; - - for RoleSection { role, section } in cases { - assert_eq!(role.opened_section(), Some(section)); - assert_eq!(role.closed_section(), None); - } - } - - #[test] - fn a_closing_role_closes_its_section_and_opens_nothing() { - let cases = [ - RoleSection { - role: MarkerRole::ReasoningClose, - section: SampledTokenSection::Reasoning, - }, - RoleSection { - role: MarkerRole::ToolCallClose, - section: SampledTokenSection::ToolCall, - }, - ]; - - for RoleSection { role, section } in cases { - assert_eq!(role.closed_section(), Some(section)); - assert_eq!(role.opened_section(), None); - } - } -} diff --git a/llama-cpp-bindings/src/marker_role_candidate.rs b/llama-cpp-bindings/src/marker_role_candidate.rs deleted file mode 100644 index 6fd437378..000000000 --- a/llama-cpp-bindings/src/marker_role_candidate.rs +++ /dev/null @@ -1,12 +0,0 @@ -use crate::marker_role::MarkerRole; -use crate::token::LlamaToken; - -#[derive(Clone, Debug, Eq, PartialEq)] -/// A tokenized marker string together with the single role it was detected as. -/// -/// Candidates are merged by token sequence, so the same tokens detected under two -/// roles become one [`crate::streaming_marker::StreamingMarker`] carrying both. -pub struct MarkerRoleCandidate { - pub tokens: Vec, - pub role: MarkerRole, -} diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index 513bb9fa5..81f6a9e2f 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -30,26 +30,16 @@ use toktrie::INVALID_TOKEN; use toktrie::TokRxInfo; use toktrie::TokTrie; -use llama_cpp_bindings_types::ReasoningMarkers; -use llama_cpp_bindings_types::ToolCallMarkers; - -use crate::bare_json_tool_calls::BareJsonToolCalls; -use crate::chat_template_tool_calls; +use crate::chat_templates::ChatTemplates; +use crate::error::chat_templates_error::ChatTemplatesError; use crate::llama_backend::LlamaBackend; use crate::llama_token_attrs::LlamaTokenAttrs; -use crate::marker_role::MarkerRole; -use crate::marker_role_candidate::MarkerRoleCandidate; use crate::model::tokenizer_input::TokenizerInput; -use crate::resolved_tool_call_markers::ResolvedToolCallMarkers; use crate::sampled_token::SampledToken; -use crate::sampled_token_classifier::SampledTokenClassifier; -use crate::streaming_markers::StreamingMarkers; -use crate::synthetic_tool_call_renders::SyntheticToolCallRenders; use crate::token::LlamaToken; use crate::{ - ApplyChatTemplateError, ChatTemplateError, LlamaLoraAdapterInitError, LlamaModelLoadError, - MarkerDetectionError, MetaValError, ParseChatMessageError, StringToTokenError, TokenAttrError, - TokenOutsideVocabulary, TokenToStringError, + ChatTemplateError, LlamaLoraAdapterInitError, LlamaModelLoadError, MetaValError, + StringToTokenError, TokenAttrError, TokenOutsideVocabulary, TokenToStringError, }; pub use add_bos::AddBos; @@ -95,77 +85,7 @@ const fn present_special_token( pub struct LlamaModel { pub model: NonNull, tok_env: OnceCell>, - chat_parser: OnceCell, - reasoning_markers: OnceCell>, - streaming_markers: OnceCell>, -} - -#[derive(Debug)] -struct ChatParserHandle { - parser: NonNull, -} - -/// # Safety -/// -/// The handle is an opaque pointer to a heap-allocated parser owned by the model; it is -/// created once, never mutated afterwards, and freed exactly once on drop. The owning -/// `LlamaModel` is already `Send + Sync`, so the handle shares that guarantee. -unsafe impl Send for ChatParserHandle {} - -unsafe impl Sync for ChatParserHandle {} - -/// # Safety -/// -/// `out_error` must be the pointer populated by the preceding -/// `llama_rs_chat_parser_free` call, or null. The destructor-threw arm reads and -/// frees it. -unsafe fn chat_parser_free_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_chat_parser_free_status, - out_error: *mut c_char, -) -> Result<(), ParseChatMessageError> { - match status { - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_FREE_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_FREE_ERROR_STRING_ALLOCATION_FAILED => { - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_FREE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - out_error, - "llama_rs_chat_parser_free", - "reported a thrown C++ exception without an error message", - ) - }?; - - Err(ParseChatMessageError::DestructorFailed { message }) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_chat_parser_free", - code: i64::from(other), - } - .into()), - } -} - -impl Drop for ChatParserHandle { - fn drop(&mut self) { - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_chat_parser_free( - self.parser.as_ptr(), - &raw mut out_error, - ) - }; - - if let Err(destructor_failure) = - unsafe { chat_parser_free_status_to_result(status, out_error) } - { - log::error!("{destructor_failure}"); - } - } + chat_templates: OnceCell, } impl std::fmt::Debug for LlamaModel { @@ -202,9 +122,7 @@ unsafe fn load_model_from_file_status_to_result( Ok(LlamaModel { model, tok_env: OnceCell::new(), - chat_parser: OnceCell::new(), - reasoning_markers: OnceCell::new(), - streaming_markers: OnceCell::new(), + chat_templates: OnceCell::new(), }) } llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_LLAMA_CPP_RETURNED_NULL => { @@ -259,161 +177,6 @@ unsafe fn load_model_from_file_status_to_result( } } -/// # Safety -/// -/// `out_error` must reference the pointer populated by the preceding -/// `llama_rs_chat_parser_create` call (or null); it is read, freed, and nulled only in the -/// CXX-exception arm. `parser` must be the pointer populated by the same call. -unsafe fn chat_parser_create_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_chat_parser_create_status, - parser: *mut llama_cpp_bindings_sys::llama_rs_chat_parser, - out_error: *mut *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_OK => NonNull::new(parser) - .map(|parser| ChatParserHandle { parser }) - .ok_or_else(|| { - crate::FfiContractError { - operation: "llama_rs_chat_parser_create", - detail: "success status contained a null parser handle", - } - .into() - }), - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_MODEL_HAS_NO_CHAT_TEMPLATE => { - Err(ParseChatMessageError::NoChatTemplate) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_MODEL_HAS_NO_VOCAB => { - Err(ParseChatMessageError::NoVocab) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_ERROR_STRING_ALLOCATION_FAILED => { - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - *out_error, - "llama_rs_chat_parser_create", - "reported a thrown C++ exception without an error message", - ) - }?; - unsafe { *out_error = ptr::null_mut() }; - Err(ParseChatMessageError::ParserCreationFailed { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_NULL_MODEL_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_chat_parser_create", - detail: "was given a null model argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_NULL_OUT_PARSER_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_chat_parser_create", - detail: "was given a null out_parser argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_chat_parser_create", - detail: "was given a null out_error argument", - } - .into()) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_chat_parser_create", - code: i64::from(other), - } - .into()), - } -} - -/// # Safety -/// -/// `out_string` and `out_error` must be the pointers populated by the preceding -/// `llama_rs_apply_chat_template` call (or null). The success arm reads and frees -/// `out_string`; the CXX-exception arm reads and frees `out_error`. -unsafe fn apply_chat_template_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_apply_chat_template_status, - out_string: *mut c_char, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_OK => Ok(unsafe { - read_and_free_cpp_string( - out_string, - "llama_rs_apply_chat_template", - "success status contained a null rendered string", - ) - }?), - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_MODEL_HAS_NO_VOCAB => { - Err(ApplyChatTemplateError::NoVocab) - } - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_TEMPLATE_APPLICATION_FAILED => { - Err(ApplyChatTemplateError::TemplateApplicationFailed) - } - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_ERROR_STRING_ALLOCATION_FAILED => { - Err(ApplyChatTemplateError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(ApplyChatTemplateError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - out_error, - "llama_rs_apply_chat_template", - "reported a thrown C++ exception without an error message", - ) - }?; - Err(ApplyChatTemplateError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_MODEL_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_apply_chat_template", - detail: "was given a null model argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_TEMPLATE_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_apply_chat_template", - detail: "was given a null template argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_MESSAGES_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_apply_chat_template", - detail: "was given a null messages argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_OUT_STRING_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_apply_chat_template", - detail: "was given a null out_string argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_apply_chat_template", - detail: "was given a null out_error argument", - } - .into()) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_apply_chat_template", - code: i64::from(other), - } - .into()), - } -} - impl LlamaModel { fn require_vocabulary_token( &self, @@ -760,21 +523,14 @@ impl LlamaModel { /// # Errors /// - /// * If the model has no chat template by that name - /// - /// # Panics - /// - /// Panics if the C-returned chat template string contains interior null bytes - /// (should never happen with valid model data). + /// Returns [`ChatTemplateError::NullError`] when `name` contains a NUL byte, or + /// [`ChatTemplateError::MissingTemplate`] when the model has no chat template by that name. pub fn chat_template( &self, name: Option<&str>, ) -> Result { - let name_cstr = name.map(CString::new); - let name_ptr = match name_cstr { - Some(Ok(name)) => name.as_ptr(), - _ => ptr::null(), - }; + let name_cstr = name.map(CString::new).transpose()?; + let name_ptr = name_cstr.as_ref().map_or(ptr::null(), |name| name.as_ptr()); let result = unsafe { llama_cpp_bindings_sys::llama_model_chat_template(self.model.as_ptr(), name_ptr) }; @@ -846,238 +602,6 @@ impl LlamaModel { llama_cpp_bindings_sys::llama_adapter_lora_init(self.model.as_ptr(), cstr.as_ptr()) }) } - - /// # Errors - /// Returns [`ApplyChatTemplateError`] if the model has no vocab, the template - /// renders an empty prompt or cannot be rendered, or the renderer throws. - pub fn apply_chat_template( - &self, - tmpl: &LlamaChatTemplate, - chat: &[LlamaChatMessage], - add_ass: bool, - enable_thinking: bool, - ) -> Result { - let roles: Vec<*const c_char> = chat - .iter() - .map(|chat_message| chat_message.role.as_ptr()) - .collect(); - let contents: Vec<*const c_char> = chat - .iter() - .map(|chat_message| chat_message.content.as_ptr()) - .collect(); - - let mut out_string: *mut c_char = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_apply_chat_template( - self.model.as_ptr(), - tmpl.0.as_ptr(), - roles.as_ptr(), - contents.as_ptr(), - chat.len(), - i32::from(add_ass), - i32::from(enable_thinking), - &raw mut out_string, - &raw mut out_error, - ) - }; - - unsafe { apply_chat_template_status_to_result(status, out_string, out_error) } - } - - /// # Errors - /// Returns [`MarkerDetectionError`] when streaming-marker detection fails. - /// The classifier is never constructed in a degraded "blind" state — a - /// detection failure is surfaced to the caller instead of silently ignored. - pub fn sampled_token_classifier( - &self, - bare_json_tool_calls: BareJsonToolCalls, - ) -> Result, MarkerDetectionError> { - self.streaming_markers() - .map(|markers| SampledTokenClassifier::new(self, markers, bare_json_tool_calls)) - } - - /// # Errors - /// Returns [`MarkerDetectionError`] when any underlying FFI call fails. - pub fn streaming_markers(&self) -> Result, MarkerDetectionError> { - self.streaming_markers - .get_or_try_init(|| self.detect_streaming_markers().map(Arc::new)) - .map(Arc::clone) - } - - fn detect_streaming_markers(&self) -> Result { - let reasoning_markers = self.reasoning_markers()?; - - let tool_call_haystack = invoke_compute_tool_call_haystack(self.model.as_ptr())?; - - let autoparser_pair = tool_call_haystack.as_deref().and_then( - crate::extract_tool_call_markers_from_haystack::extract_tool_call_markers_from_haystack, - ); - - let (autoparser_open, autoparser_close) = match autoparser_pair { - Some(crate::tool_call_marker_pair::ToolCallMarkerPair { open, close }) => { - (Some(open), Some(close)) - } - None => (None, None), - }; - - let resolved_tool_call_markers = - self.resolve_tool_call_marker_strings(autoparser_open, autoparser_close)?; - - let mut candidates = Vec::new(); - if let Some(markers) = reasoning_markers { - for marker in &markers.closes { - if let Some(tokens) = self.tokenize_marker(Some(marker))? { - candidates.push(MarkerRoleCandidate { - tokens, - role: MarkerRole::ReasoningClose, - }); - } - } - } - - let reasoning_open = reasoning_markers.map(|markers| markers.open.as_str()); - if let Some(tokens) = self.tokenize_marker(reasoning_open)? { - candidates.push(MarkerRoleCandidate { - tokens, - role: MarkerRole::ReasoningOpen, - }); - } - if let Some(tokens) = self.tokenize_marker(resolved_tool_call_markers.open.as_deref())? { - candidates.push(MarkerRoleCandidate { - tokens, - role: MarkerRole::ToolCallOpen, - }); - } - if let Some(tokens) = self.tokenize_marker(resolved_tool_call_markers.close.as_deref())? { - candidates.push(MarkerRoleCandidate { - tokens, - role: MarkerRole::ToolCallClose, - }); - } - - StreamingMarkers::from_candidates(candidates) - } - - fn resolve_tool_call_marker_strings( - &self, - autoparser_open: Option, - autoparser_close: Option, - ) -> Result { - if autoparser_open - .as_deref() - .is_some_and(|raw| !raw.trim().is_empty()) - { - return Ok(ResolvedToolCallMarkers { - open: autoparser_open, - close: autoparser_close, - }); - } - let Some(markers) = self.tool_call_markers()? else { - return Ok(ResolvedToolCallMarkers { - open: autoparser_open, - close: autoparser_close, - }); - }; - let close = if markers.close.is_empty() { - None - } else { - Some(markers.close) - }; - Ok(ResolvedToolCallMarkers { - open: Some(markers.open), - close, - }) - } - - /// # Errors - /// Returns [`MarkerDetectionError`] when the underlying FFI call fails. - pub fn reasoning_markers(&self) -> Result, MarkerDetectionError> { - self.reasoning_markers - .get_or_try_init(|| invoke_detect_reasoning_markers(self.model.as_ptr())) - .map(Option::as_ref) - } - - /// # Errors - /// Returns [`MarkerDetectionError::ToolCallTemplateNotUtf8`] when the model - /// has a chat template that is not valid UTF-8. A model with no chat - /// template legitimately yields `Ok(None)`. - pub fn tool_call_markers(&self) -> Result, MarkerDetectionError> { - let template = match self.chat_template(None) { - Ok(template) => template, - Err(ChatTemplateError::MissingTemplate) => return Ok(None), - Err(other) => return Err(MarkerDetectionError::ChatTemplateUnavailable(other)), - }; - let template_str = template.to_str()?; - - Ok(chat_template_tool_calls::detect(template_str)) - } - - /// # Errors - /// Returns [`StringToTokenError`] when a present, non-empty marker string - /// fails to tokenise. - fn tokenize_marker( - &self, - marker: Option<&str>, - ) -> Result>, StringToTokenError> { - let Some(marker) = marker else { - return Ok(None); - }; - let marker = marker.trim(); - if marker.is_empty() { - return Ok(None); - } - let tokens = self.str_to_token(marker, AddBos::Never, ParseSpecialTokens::Always)?; - if tokens.is_empty() { - Ok(None) - } else { - Ok(Some(tokens)) - } - } - - /// Returns the model's chat parser, analysing the chat template on first use. The - /// pointer stays valid for as long as the model lives. - /// - /// # Errors - /// - /// Returns [`ParseChatMessageError`] when the model has no chat template or vocab, or - /// the parser cannot be constructed. - pub fn chat_parser_ptr( - &self, - ) -> Result<*mut llama_cpp_bindings_sys::llama_rs_chat_parser, ParseChatMessageError> { - self.chat_parser().map(|handle| handle.parser.as_ptr()) - } - - fn chat_parser(&self) -> Result<&ChatParserHandle, ParseChatMessageError> { - self.chat_parser - .get_or_try_init(|| self.create_chat_parser()) - } - - fn create_chat_parser(&self) -> Result { - let mut out_parser: *mut llama_cpp_bindings_sys::llama_rs_chat_parser = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_chat_parser_create( - self.model.as_ptr(), - &raw mut out_parser, - &raw mut out_error, - ) - }; - - unsafe { chat_parser_create_status_to_result(status, out_parser, &raw mut out_error) } - } - - /// # Errors - /// - /// Returns [`MarkerDetectionError`] when the C++ analyzer throws or the FFI - /// returns a non-OK status. - pub fn diagnose_tool_call_synthetic_renders( - &self, - ) -> Result { - invoke_diagnose_tool_call_synthetic_renders(self.model.as_ptr()) - } } fn initialize_lora_adapter<'model>( @@ -1098,6 +622,17 @@ impl LlamaModel { .get_or_try_init(|| build_approximate_tok_env(self)) .map(Arc::clone) } + + /// Returns the model's chat templates, preparing them on first use. + /// + /// # Errors + /// + /// Returns [`ChatTemplatesError`] when the model has no chat template or llama.cpp cannot + /// prepare it. + pub fn chat_templates(&self) -> Result<&ChatTemplates, ChatTemplatesError> { + self.chat_templates + .get_or_try_init(|| ChatTemplates::new(self)) + } } const TOK_ENV_PIECE_PROBE_SIZE: usize = 32; @@ -1155,432 +690,48 @@ fn build_approximate_tok_env( Ok(Arc::new(ApproximateTokEnv::new(trie))) } -unsafe fn detect_reasoning_markers_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_detect_reasoning_markers_status, - out_markers: *const llama_cpp_bindings_sys::llama_rs_reasoning_markers, +/// # Safety +/// +/// `out_error` must be the pointer populated by the preceding `llama_rs_tokenize` call (or +/// null when no error was produced); it is read and freed only in the CXX-exception arm. +unsafe fn tokenize_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_tokenize_status, + out_count: c_int, out_error: *mut c_char, -) -> Result, MarkerDetectionError> { +) -> Result { match status { - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_OK => unsafe { - read_reasoning_markers(out_markers) - }, - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_NULL_MODEL_ARG => { - Err(MarkerDetectionError::WrapperRejectedArgument { - operation: "llama_rs_detect_reasoning_markers", - argument: "model", - }) - } - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_MARKERS_ARG => { - Err(MarkerDetectionError::WrapperRejectedArgument { - operation: "llama_rs_detect_reasoning_markers", - argument: "out_markers", - }) - } - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_ERROR_ARG => { - Err(MarkerDetectionError::WrapperRejectedArgument { - operation: "llama_rs_detect_reasoning_markers", - argument: "out_error", - }) + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_OK => Ok(out_count), + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_ERROR_STRING_ALLOCATION_FAILED => { + Err(StringToTokenError::NotEnoughMemory) } - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_MODEL_HAS_NO_CHAT_TEMPLATE => Ok(None), - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_MODEL_HAS_NO_VOCAB => { - Err(MarkerDetectionError::ModelHasNoVocab { - operation: "llama_rs_detect_reasoning_markers", - }) + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_LLAMA_CPP_OUT_OF_MEMORY => { + Err(StringToTokenError::LlamaCppOutOfMemory) } - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_ERROR_STRING_ALLOCATION_FAILED => { - Err(MarkerDetectionError::NotEnoughMemory) + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + "llama_rs_tokenize", + "reported a thrown C++ exception without an error message", + ) + }?; + Err(StringToTokenError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_LLAMA_CPP_OUT_OF_MEMORY => { - Err(MarkerDetectionError::LlamaCppOutOfMemory) + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_VOCAB_ARG => { + unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; + Err(crate::FfiContractError { + operation: "llama_rs_tokenize", + detail: "was given a null vocab argument", + } + .into()) } - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { read_and_free_cpp_string(out_error, "llama_rs_detect_reasoning_markers", "reported a thrown C++ exception without an error message") }?; - Err(MarkerDetectionError::ReasoningMarkerDetectionFailed { message }) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_detect_reasoning_markers", - code: i64::from(other), - } - .into()), - } -} - -unsafe fn read_reasoning_markers( - markers: *const llama_cpp_bindings_sys::llama_rs_reasoning_markers, -) -> Result, MarkerDetectionError> { - if markers.is_null() { - return Ok(None); - } - let open_pointer = unsafe { llama_cpp_bindings_sys::llama_rs_reasoning_markers_open(markers) }; - let open = read_optional_owned_cstr(open_pointer)?; - let close_count = - unsafe { llama_cpp_bindings_sys::llama_rs_reasoning_markers_close_count(markers) }; - let mut closes = Vec::with_capacity(close_count); - for index in 0..close_count { - let close_pointer = - unsafe { llama_cpp_bindings_sys::llama_rs_reasoning_markers_close_at(markers, index) }; - closes.push(read_optional_owned_cstr(close_pointer)?); - } - validate_reasoning_markers(open, closes).map(Some) -} - -fn validate_reasoning_markers( - open: Option, - closes: Vec>, -) -> Result { - let Some(open) = open else { - return Err(crate::FfiContractError { - operation: "llama_rs_reasoning_markers_open", - detail: "non-null markers returned a null opening marker", - } - .into()); - }; - if open.is_empty() || closes.is_empty() { - return Err(crate::FfiContractError { - operation: "llama_rs_detect_reasoning_markers", - detail: "detected markers must contain an opening marker and a closing marker", - } - .into()); - } - let closes = closes - .into_iter() - .map(|close| { - let Some(close) = close else { - return Err(crate::FfiContractError { - operation: "llama_rs_reasoning_markers_close_at", - detail: "a valid closing-marker index returned null", - } - .into()); - }; - if close.is_empty() { - return Err(crate::FfiContractError { - operation: "llama_rs_reasoning_markers_close_at", - detail: "a detected closing marker was empty", - } - .into()); - } - Ok(close) - }) - .collect::, MarkerDetectionError>>()?; - Ok(ReasoningMarkers { open, closes }) -} - -const fn cxx_exception_owns_out_error( - parsed: &Result, -) -> bool { - matches!( - parsed, - Err(MarkerDetectionError::ReasoningMarkerDetectionFailed { .. } - | MarkerDetectionError::ToolCallHaystackComputationFailed { .. } - | MarkerDetectionError::ToolCallSyntheticRenderDiagnosisFailed { .. }) - ) -} - -/// # Safety -/// -/// `free_error` must be the pointer populated by the preceding -/// `llama_rs_reasoning_markers_free` call, or null. The destructor-threw arm -/// reads and frees it. -unsafe fn reasoning_markers_free_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_reasoning_markers_free_status, - free_error: *mut c_char, -) -> Result<(), MarkerDetectionError> { - match status { - llama_cpp_bindings_sys::LLAMA_RS_REASONING_MARKERS_FREE_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_REASONING_MARKERS_FREE_ERROR_STRING_ALLOCATION_FAILED => { - Err(MarkerDetectionError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_REASONING_MARKERS_FREE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(MarkerDetectionError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_REASONING_MARKERS_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - free_error, - "llama_rs_reasoning_markers_free", - "reported a thrown C++ exception without an error message", - ) - }?; - - Err(MarkerDetectionError::ReasoningMarkersFreeFailed { message }) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_reasoning_markers_free", - code: i64::from(other), - } - .into()), - } -} - -fn invoke_detect_reasoning_markers( - model: *const llama_cpp_bindings_sys::llama_model, -) -> Result, MarkerDetectionError> { - let mut out_markers: *mut llama_cpp_bindings_sys::llama_rs_reasoning_markers = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_detect_reasoning_markers( - model, - &raw mut out_markers, - &raw mut out_error, - ) - }; - - let parsed = - unsafe { detect_reasoning_markers_status_to_result(status, out_markers, out_error) }; - - let mut free_error: *mut c_char = ptr::null_mut(); - let free_status = unsafe { - llama_cpp_bindings_sys::llama_rs_reasoning_markers_free(out_markers, &raw mut free_error) - }; - let freed = unsafe { reasoning_markers_free_status_to_result(free_status, free_error) }; - - if !cxx_exception_owns_out_error(&parsed) { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - } - - match parsed { - Ok(markers) => freed.map(|()| markers), - Err(detection_failure) => { - if let Err(destructor_failure) = freed { - log::error!("{destructor_failure}"); - } - - Err(detection_failure) - } - } -} - -/// # Safety -/// -/// `out_haystack` and `out_error` must be the pointers populated by the preceding -/// `llama_rs_compute_tool_call_haystack` call (or null). `out_haystack` is read but not freed -/// here; `out_error` is freed only in the CXX-exception arm, mirroring the conditional cleanup -/// in the caller. -unsafe fn compute_tool_call_haystack_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_compute_tool_call_haystack_status, - out_haystack: *const c_char, - out_error: *mut c_char, -) -> Result, MarkerDetectionError> { - match status { - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_OK => { - read_optional_owned_cstr(out_haystack) - } - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_MODEL_HAS_NO_CHAT_TEMPLATE => Ok(None), - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_MODEL_HAS_NO_VOCAB => { - Err(MarkerDetectionError::ModelHasNoVocab { - operation: "llama_rs_compute_tool_call_haystack", - }) - } - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_ERROR_STRING_ALLOCATION_FAILED => { - Err(MarkerDetectionError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_LLAMA_CPP_OUT_OF_MEMORY => { - Err(MarkerDetectionError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { read_and_free_cpp_string(out_error, "llama_rs_compute_tool_call_haystack", "reported a thrown C++ exception without an error message") }?; - Err(MarkerDetectionError::ToolCallHaystackComputationFailed { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_MODEL_ARG => Err(crate::FfiContractError { - operation: "llama_rs_compute_tool_call_haystack", - detail: "was given a null model argument", - } - .into()), - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_OUT_HAYSTACK_ARG => Err(crate::FfiContractError { - operation: "llama_rs_compute_tool_call_haystack", - detail: "was given a null out_haystack argument", - } - .into()), - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_OUT_ERROR_ARG => Err(crate::FfiContractError { - operation: "llama_rs_compute_tool_call_haystack", - detail: "was given a null out_error argument", - } - .into()), - other => Err(crate::FfiStatusError { - operation: "llama_rs_compute_tool_call_haystack", - code: i64::from(other), - } - .into()), - } -} - -fn invoke_compute_tool_call_haystack( - model: *const llama_cpp_bindings_sys::llama_model, -) -> Result, MarkerDetectionError> { - let mut out_haystack: *mut c_char = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_compute_tool_call_haystack( - model, - &raw mut out_haystack, - &raw mut out_error, - ) - }; - - let parsed = - unsafe { compute_tool_call_haystack_status_to_result(status, out_haystack, out_error) }; - - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_haystack) }; - if !cxx_exception_owns_out_error(&parsed) { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - } - - parsed -} - -/// # Safety -/// -/// `out_no_tools`, `out_with_tools`, and `out_error` must be the pointers populated by the -/// preceding `llama_rs_diagnose_tool_call_synthetic_renders` call (or null). The render -/// pointers are read but not freed here; `out_error` is freed only in the CXX-exception arm, -/// mirroring the cleanup in the caller. -unsafe fn diagnose_tool_call_synthetic_renders_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_diagnose_tool_call_synthetic_renders_status, - out_no_tools: *const c_char, - out_with_tools: *const c_char, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_OK => { - collect_synthetic_tool_call_renders(out_no_tools, out_with_tools) - } - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_MODEL_HAS_NO_CHAT_TEMPLATE => { - Err(MarkerDetectionError::ModelHasNoChatTemplate { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - }) - } - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_MODEL_HAS_NO_VOCAB => { - Err(MarkerDetectionError::ModelHasNoVocab { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - }) - } - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_ERROR_STRING_ALLOCATION_FAILED => { - Err(MarkerDetectionError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_LLAMA_CPP_OUT_OF_MEMORY => { - Err(MarkerDetectionError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { read_and_free_cpp_string(out_error, "llama_rs_diagnose_tool_call_synthetic_renders", "reported a thrown C++ exception without an error message") }?; - Err(MarkerDetectionError::ToolCallSyntheticRenderDiagnosisFailed { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_MODEL_ARG => Err(crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "was given a null model argument", - } - .into()), - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_NO_TOOLS_ARG => Err(crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "was given a null out_no_tools argument", - } - .into()), - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_WITH_TOOLS_ARG => Err(crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "was given a null out_with_tools argument", - } - .into()), - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_ERROR_ARG => Err(crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "was given a null out_error argument", - } - .into()), - other => Err(crate::FfiStatusError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - code: i64::from(other), - } - .into()), - } -} - -fn invoke_diagnose_tool_call_synthetic_renders( - model: *const llama_cpp_bindings_sys::llama_model, -) -> Result { - let mut out_no_tools: *mut c_char = ptr::null_mut(); - let mut out_with_tools: *mut c_char = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_diagnose_tool_call_synthetic_renders( - model, - &raw mut out_no_tools, - &raw mut out_with_tools, - &raw mut out_error, - ) - }; - - let parsed = unsafe { - diagnose_tool_call_synthetic_renders_status_to_result( - status, - out_no_tools, - out_with_tools, - out_error, - ) - }; - - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_no_tools) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_with_tools) }; - if !cxx_exception_owns_out_error(&parsed) { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - } - - parsed -} - -fn read_optional_owned_cstr(ptr: *const c_char) -> Result, MarkerDetectionError> { - if ptr.is_null() { - return Ok(None); - } - - let bytes = unsafe { CStr::from_ptr(ptr) }.to_bytes().to_vec(); - - Ok(Some(String::from_utf8(bytes)?)) -} - -/// # Safety -/// -/// `out_error` must be the pointer populated by the preceding `llama_rs_tokenize` call (or -/// null when no error was produced); it is read and freed only in the CXX-exception arm. -unsafe fn tokenize_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_tokenize_status, - out_count: c_int, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_OK => Ok(out_count), - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_ERROR_STRING_ALLOCATION_FAILED => { - Err(StringToTokenError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(StringToTokenError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - out_error, - "llama_rs_tokenize", - "reported a thrown C++ exception without an error message", - ) - }?; - Err(StringToTokenError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_VOCAB_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_tokenize", - detail: "was given a null vocab argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_TEXT_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_tokenize", - detail: "was given a null text argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_TEXT_ARG => { + unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; + Err(crate::FfiContractError { + operation: "llama_rs_tokenize", + detail: "was given a null text argument", + } + .into()) } llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_OUT_RETURNED_COUNT_ARG => { unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; @@ -1680,26 +831,6 @@ fn tokenize_into_buffer( Ok(buffer) } -fn collect_synthetic_tool_call_renders( - without_tools_ptr: *const c_char, - with_tools_ptr: *const c_char, -) -> Result { - let without_tools = - read_optional_owned_cstr(without_tools_ptr)?.ok_or(crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "success status contained a null render without tools", - })?; - let with_tools = read_optional_owned_cstr(with_tools_ptr)?.ok_or(crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "success status contained a null render with tools", - })?; - - Ok(SyntheticToolCallRenders { - without_tools, - with_tools, - }) -} - fn extract_meta_string( c_function: TCFunction, capacity: usize, @@ -1857,95 +988,6 @@ mod tokenize_into_buffer_tests { } } -#[cfg(test)] -mod read_optional_owned_cstr_tests { - - use super::read_optional_owned_cstr; - use crate::MarkerDetectionError; - - #[test] - fn read_optional_owned_cstr_invalid_utf8_returns_error() { - let invalid_utf8_with_terminator: [u8; 3] = [0xFF, 0xFE, 0x00]; - let result = read_optional_owned_cstr( - invalid_utf8_with_terminator - .as_ptr() - .cast::(), - ); - - assert_eq!( - result.unwrap_err(), - MarkerDetectionError::MarkerUtf8Error(String::from_utf8(vec![0xFF, 0xFE]).unwrap_err()) - ); - } -} - -#[cfg(test)] -mod collect_synthetic_tool_call_renders_tests { - - use super::collect_synthetic_tool_call_renders; - use crate::MarkerDetectionError; - - #[test] - fn collect_synthetic_tool_call_renders_first_invalid_utf8_returns_error() { - let invalid_utf8_with_terminator: [u8; 3] = [0xFF, 0xFE, 0x00]; - let valid_with_terminator: [u8; 3] = [b'o', b'k', 0x00]; - let result = collect_synthetic_tool_call_renders( - invalid_utf8_with_terminator - .as_ptr() - .cast::(), - valid_with_terminator.as_ptr().cast::(), - ); - - assert_eq!( - result.unwrap_err(), - MarkerDetectionError::MarkerUtf8Error(String::from_utf8(vec![0xFF, 0xFE]).unwrap_err()) - ); - } - - #[test] - fn collect_synthetic_tool_call_renders_second_invalid_utf8_returns_error() { - let valid_with_terminator: [u8; 3] = [b'o', b'k', 0x00]; - let invalid_utf8_with_terminator: [u8; 3] = [0xFF, 0xFE, 0x00]; - let result = collect_synthetic_tool_call_renders( - valid_with_terminator.as_ptr().cast::(), - invalid_utf8_with_terminator - .as_ptr() - .cast::(), - ); - - assert_eq!( - result.unwrap_err(), - MarkerDetectionError::MarkerUtf8Error(String::from_utf8(vec![0xFF, 0xFE]).unwrap_err()) - ); - } -} - -#[cfg(test)] -mod cxx_exception_owns_out_error_tests { - use super::cxx_exception_owns_out_error; - use crate::MarkerDetectionError; - - #[test] - fn cxx_exception_owns_out_error_classifies_each_failure_variant() { - assert!(cxx_exception_owns_out_error::<()>(&Err( - MarkerDetectionError::ReasoningMarkerDetectionFailed { - message: String::new() - } - ))); - assert!(cxx_exception_owns_out_error::<()>(&Err( - MarkerDetectionError::ToolCallHaystackComputationFailed { - message: String::new() - } - ))); - assert!(cxx_exception_owns_out_error::<()>(&Err( - MarkerDetectionError::ToolCallSyntheticRenderDiagnosisFailed { - message: String::new() - } - ))); - assert!(!cxx_exception_owns_out_error::<()>(&Ok(()))); - } -} - #[cfg(test)] mod extract_meta_string_tests { use super::extract_meta_string; @@ -1989,8 +1031,9 @@ mod extract_meta_string_tests { 4, ); - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("FromUtf8Error")); + let invalid_utf8 = String::from_utf8(vec![0xFF, 0xFE]).unwrap_err(); + + assert_eq!(result, Err(MetaValError::FromUtf8Error(invalid_utf8))); } #[test] @@ -2233,1059 +1276,77 @@ mod load_model_from_file_status_to_result_tests { } #[cfg(test)] -mod chat_parser_create_status_to_result_tests { - use std::ffi::c_char; - use std::mem::discriminant; +mod tokenize_status_to_result_tests { use std::ptr; - use super::chat_parser_create_status_to_result; - use crate::ParseChatMessageError; + use super::tokenize_status_to_result; + use crate::StringToTokenError; #[test] - fn chat_parser_create_no_chat_template_maps_to_no_chat_template() { - let mut out_error: *mut c_char = ptr::null_mut(); + fn tokenize_ok_returns_count() { let result = unsafe { - chat_parser_create_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_MODEL_HAS_NO_CHAT_TEMPLATE, + tokenize_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_OK, + 12, ptr::null_mut(), - &raw mut out_error, ) }; - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::NoChatTemplate) - ); + assert_eq!(result, Ok(12)); } #[test] - fn chat_parser_create_no_vocab_maps_to_no_vocab() { - let mut out_error: *mut c_char = ptr::null_mut(); + fn tokenize_allocation_failed_is_not_enough_memory() { let result = unsafe { - chat_parser_create_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_MODEL_HAS_NO_VOCAB, + tokenize_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_ERROR_STRING_ALLOCATION_FAILED, + 0, ptr::null_mut(), - &raw mut out_error, ) }; - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::NoVocab) - ); + assert_eq!(result, Err(StringToTokenError::NotEnoughMemory)); } #[test] - fn chat_parser_create_allocation_failed_is_not_enough_memory() { - let mut out_error: *mut c_char = ptr::null_mut(); + fn tokenize_cxx_exception_without_a_message_is_a_contract_error() { let result = unsafe { - chat_parser_create_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_ERROR_STRING_ALLOCATION_FAILED, + tokenize_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, ptr::null_mut(), - &raw mut out_error, - ) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::NotEnoughMemory) - ); - } - - #[test] - fn chat_parser_create_cxx_exception_is_parser_creation_failed_and_nulls_error() { - let mut out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"the parser could not be built".as_ptr()) - }; - let result = unsafe { - chat_parser_create_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null_mut(), - &raw mut out_error, - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::ParserCreationFailed { - message: "the parser could not be built".to_owned() - }) - ); - - assert!( - out_error.is_null(), - "the reclaimed pointer must be nulled so the caller does not free it twice" - ); - } - - #[test] - fn chat_parser_create_success_with_null_parser_is_contract_error() { - let mut out_error: *mut c_char = ptr::null_mut(); - let result = unsafe { - chat_parser_create_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_OK, - ptr::null_mut(), - &raw mut out_error, - ) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_chat_parser_create", - detail: "success status contained a null parser handle", - } - )) - ); - } - - #[test] - fn chat_parser_create_unknown_status_is_preserved() { - let mut out_error: *mut c_char = ptr::null_mut(); - let result = unsafe { - chat_parser_create_status_to_result(255, ptr::null_mut(), &raw mut out_error) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_chat_parser_create", - code: 255, - })) - ); - } - - #[test] - fn chat_parser_create_status_to_result_maps_null_model_arg() { - let mut out_error_slot: *mut c_char = ptr::null_mut(); - let outcome = unsafe { - chat_parser_create_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_NULL_MODEL_ARG, - ptr::null_mut(), - &raw mut out_error_slot, - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_chat_parser_create", - detail: "was given a null model argument", - } - )) - ); - } - - #[test] - fn chat_parser_create_status_to_result_maps_null_out_parser_arg() { - let mut out_error_slot: *mut c_char = ptr::null_mut(); - let outcome = unsafe { - chat_parser_create_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_NULL_OUT_PARSER_ARG, - ptr::null_mut(), - &raw mut out_error_slot, - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_chat_parser_create", - detail: "was given a null out_parser argument", - } - )) - ); - } - - #[test] - fn chat_parser_create_status_to_result_maps_null_out_error_arg() { - let mut out_error_slot: *mut c_char = ptr::null_mut(); - let outcome = unsafe { - chat_parser_create_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_NULL_OUT_ERROR_ARG, - ptr::null_mut(), - &raw mut out_error_slot, - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_chat_parser_create", - detail: "was given a null out_error argument", - } - )) - ); - } - - #[test] - fn chat_parser_create_status_to_result_maps_llama_cpp_out_of_memory() { - let mut out_error_slot: *mut c_char = ptr::null_mut(); - let outcome = unsafe { - chat_parser_create_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null_mut(), - &raw mut out_error_slot, - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::LlamaCppOutOfMemory) - ); - } -} - -#[cfg(test)] -mod detect_reasoning_markers_status_to_result_tests { - use std::ptr; - - use super::detect_reasoning_markers_status_to_result; - use crate::MarkerDetectionError; - - #[test] - fn detect_reasoning_markers_ok_with_null_handle_returns_none() { - let result = unsafe { - detect_reasoning_markers_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_OK, - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!(result, Ok(None)); - } - - #[test] - fn detect_reasoning_markers_allocation_failed_is_not_enough_memory() { - let result = unsafe { - detect_reasoning_markers_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_ERROR_STRING_ALLOCATION_FAILED, - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!(result, Err(MarkerDetectionError::NotEnoughMemory)); - } - - #[test] - fn detect_reasoning_markers_cxx_exception_without_a_message_is_a_contract_error() { - let result = unsafe { - detect_reasoning_markers_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result, - Err(crate::FfiContractError { - operation: "llama_rs_detect_reasoning_markers", - detail: "reported a thrown C++ exception without an error message", - } - .into()) - ); - } - - #[test] - fn detect_reasoning_markers_unknown_status_is_preserved() { - let result = - unsafe { detect_reasoning_markers_status_to_result(255, ptr::null(), ptr::null_mut()) }; - - assert_eq!( - result, - Err(MarkerDetectionError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_detect_reasoning_markers", - code: 255, - })) - ); - } - - #[test] - fn detect_reasoning_markers_null_model_status_names_model_argument() { - let result = unsafe { - detect_reasoning_markers_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_NULL_MODEL_ARG, - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result, - Err(MarkerDetectionError::WrapperRejectedArgument { - operation: "llama_rs_detect_reasoning_markers", - argument: "model", - }) - ); - } - - #[test] - fn detect_reasoning_markers_null_output_status_names_markers_argument() { - let result = unsafe { - detect_reasoning_markers_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_MARKERS_ARG, - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result, - Err(MarkerDetectionError::WrapperRejectedArgument { - operation: "llama_rs_detect_reasoning_markers", - argument: "out_markers", - }) - ); - } - - #[test] - fn detect_reasoning_markers_null_error_status_names_error_argument() { - let result = unsafe { - detect_reasoning_markers_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_ERROR_ARG, - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result, - Err(MarkerDetectionError::WrapperRejectedArgument { - operation: "llama_rs_detect_reasoning_markers", - argument: "out_error", - }) - ); - } - - #[test] - fn detect_reasoning_markers_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = unsafe { - detect_reasoning_markers_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(MarkerDetectionError::LlamaCppOutOfMemory) - ); - } -} - -#[cfg(test)] -mod validate_reasoning_markers_tests { - use llama_cpp_bindings_types::ReasoningMarkers; - - use super::validate_reasoning_markers; - use crate::MarkerDetectionError; - - #[test] - fn reasoning_markers_reject_a_missing_opening_marker() { - let result = validate_reasoning_markers(None, vec![Some("".to_owned())]); - - assert_eq!( - result, - Err(MarkerDetectionError::FfiContract(crate::FfiContractError { - operation: "llama_rs_reasoning_markers_open", - detail: "non-null markers returned a null opening marker", - })) - ); - } - - #[test] - fn reasoning_markers_reject_an_empty_opening_marker() { - let result = - validate_reasoning_markers(Some(String::new()), vec![Some("".to_owned())]); - - assert_eq!( - result, - Err(MarkerDetectionError::FfiContract(crate::FfiContractError { - operation: "llama_rs_detect_reasoning_markers", - detail: "detected markers must contain an opening marker and a closing marker", - })) - ); - } - - #[test] - fn reasoning_markers_reject_an_empty_closing_marker_collection() { - let result = validate_reasoning_markers(Some("".to_owned()), Vec::new()); - - assert_eq!( - result, - Err(MarkerDetectionError::FfiContract(crate::FfiContractError { - operation: "llama_rs_detect_reasoning_markers", - detail: "detected markers must contain an opening marker and a closing marker", - })) - ); - } - - #[test] - fn reasoning_markers_reject_a_missing_closing_marker() { - let result = validate_reasoning_markers(Some("".to_owned()), vec![None]); - - assert_eq!( - result, - Err(MarkerDetectionError::FfiContract(crate::FfiContractError { - operation: "llama_rs_reasoning_markers_close_at", - detail: "a valid closing-marker index returned null", - })) - ); - } - - #[test] - fn reasoning_markers_reject_an_empty_closing_marker() { - let result = - validate_reasoning_markers(Some("".to_owned()), vec![Some(String::new())]); - - assert_eq!( - result, - Err(MarkerDetectionError::FfiContract(crate::FfiContractError { - operation: "llama_rs_reasoning_markers_close_at", - detail: "a detected closing marker was empty", - })) - ); - } - - #[test] - fn reasoning_markers_preserve_every_valid_closing_marker() { - let result = validate_reasoning_markers( - Some("".to_owned()), - vec![Some("".to_owned()), Some("".to_owned())], - ); - - assert_eq!( - result, - Ok(ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned(), "".to_owned()], - }) - ); - } -} - -#[cfg(test)] -mod compute_tool_call_haystack_status_to_result_tests { - use std::ptr; - - use super::compute_tool_call_haystack_status_to_result; - use crate::MarkerDetectionError; - - #[test] - fn compute_tool_call_haystack_ok_with_null_pointer_is_none() { - let result = unsafe { - compute_tool_call_haystack_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_OK, - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!(result, Ok(None)); - } - - #[test] - fn compute_tool_call_haystack_allocation_failed_is_not_enough_memory() { - let result = unsafe { - compute_tool_call_haystack_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_ERROR_STRING_ALLOCATION_FAILED, - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!(result, Err(MarkerDetectionError::NotEnoughMemory)); - } - - #[test] - fn compute_tool_call_haystack_cxx_exception_without_a_message_is_a_contract_error() { - let result = unsafe { - compute_tool_call_haystack_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result, - Err(crate::FfiContractError { - operation: "llama_rs_compute_tool_call_haystack", - detail: "reported a thrown C++ exception without an error message", - } - .into()) - ); - } - - #[test] - fn compute_tool_call_haystack_unknown_status_is_preserved() { - let result = unsafe { - compute_tool_call_haystack_status_to_result(255, ptr::null(), ptr::null_mut()) - }; - - assert_eq!( - result, - Err(MarkerDetectionError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_compute_tool_call_haystack", - code: 255, - })) - ); - } - - #[test] - fn compute_tool_call_haystack_status_to_result_maps_null_model_arg() { - let outcome = unsafe { - compute_tool_call_haystack_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_MODEL_ARG, - ptr::null(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_compute_tool_call_haystack", - detail: "was given a null model argument", - } - .into() - ) - ); - } - - #[test] - fn compute_tool_call_haystack_status_to_result_maps_null_out_haystack_arg() { - let outcome = unsafe { - compute_tool_call_haystack_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_OUT_HAYSTACK_ARG, - ptr::null(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_compute_tool_call_haystack", - detail: "was given a null out_haystack argument", - } - .into() - ) - ); - } - - #[test] - fn compute_tool_call_haystack_status_to_result_maps_null_out_error_arg() { - let outcome = unsafe { - compute_tool_call_haystack_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_OUT_ERROR_ARG, - ptr::null(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_compute_tool_call_haystack", - detail: "was given a null out_error argument", - } - .into() - ) - ); - } - - #[test] - fn compute_tool_call_haystack_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = unsafe { - compute_tool_call_haystack_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(MarkerDetectionError::LlamaCppOutOfMemory) - ); - } -} - -#[cfg(test)] -mod diagnose_tool_call_synthetic_renders_status_to_result_tests { - use std::ptr; - - use super::diagnose_tool_call_synthetic_renders_status_to_result; - use crate::MarkerDetectionError; - - #[test] - fn diagnose_tool_call_synthetic_renders_ok_with_null_pointers_is_a_contract_error() { - let result = unsafe { - diagnose_tool_call_synthetic_renders_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_OK, - ptr::null(), - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result, - Err(crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "success status contained a null render without tools", - } - .into()) - ); - } - - #[test] - fn diagnose_tool_call_synthetic_renders_allocation_failed_is_not_enough_memory() { - let result = unsafe { - diagnose_tool_call_synthetic_renders_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_ERROR_STRING_ALLOCATION_FAILED, - ptr::null(), - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!(result, Err(MarkerDetectionError::NotEnoughMemory)); - } - - #[test] - fn diagnose_tool_call_synthetic_renders_cxx_exception_without_a_message_is_a_contract_error() { - let result = unsafe { - diagnose_tool_call_synthetic_renders_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null(), - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result, - Err(crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "reported a thrown C++ exception without an error message", - } - .into()) - ); - } - - #[test] - fn diagnose_tool_call_synthetic_renders_unknown_status_is_preserved() { - let result = unsafe { - diagnose_tool_call_synthetic_renders_status_to_result( - 255, - ptr::null(), - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result, - Err(MarkerDetectionError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - code: 255, - })) - ); - } - - #[test] - fn diagnose_tool_call_synthetic_renders_status_to_result_maps_null_model_arg() { - let outcome = unsafe { - diagnose_tool_call_synthetic_renders_status_to_result(llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_MODEL_ARG, ptr::null(), ptr::null(), ptr::null_mut()) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "was given a null model argument", - } - .into() - ) - ); - } - - #[test] - fn diagnose_tool_call_synthetic_renders_status_to_result_maps_null_out_no_tools_arg() { - let outcome = unsafe { - diagnose_tool_call_synthetic_renders_status_to_result(llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_NO_TOOLS_ARG, ptr::null(), ptr::null(), ptr::null_mut()) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "was given a null out_no_tools argument", - } - .into() - ) - ); - } - - #[test] - fn diagnose_tool_call_synthetic_renders_status_to_result_maps_null_out_with_tools_arg() { - let outcome = unsafe { - diagnose_tool_call_synthetic_renders_status_to_result(llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_WITH_TOOLS_ARG, ptr::null(), ptr::null(), ptr::null_mut()) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "was given a null out_with_tools argument", - } - .into() - ) - ); - } - - #[test] - fn diagnose_tool_call_synthetic_renders_status_to_result_maps_null_out_error_arg() { - let outcome = unsafe { - diagnose_tool_call_synthetic_renders_status_to_result(llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_ERROR_ARG, ptr::null(), ptr::null(), ptr::null_mut()) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "was given a null out_error argument", - } - .into() - ) - ); - } - - #[test] - fn diagnose_tool_call_synthetic_renders_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = unsafe { - diagnose_tool_call_synthetic_renders_status_to_result(llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_LLAMA_CPP_OUT_OF_MEMORY, ptr::null(), ptr::null(), ptr::null_mut()) - }; - assert_eq!( - outcome.err(), - Some(MarkerDetectionError::LlamaCppOutOfMemory) - ); - } -} - -#[cfg(test)] -mod tokenize_status_to_result_tests { - use std::ptr; - - use super::tokenize_status_to_result; - use crate::StringToTokenError; - - #[test] - fn tokenize_ok_returns_count() { - let result = unsafe { - tokenize_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_OK, - 12, - ptr::null_mut(), - ) - }; - - assert_eq!(result, Ok(12)); - } - - #[test] - fn tokenize_allocation_failed_is_not_enough_memory() { - let result = unsafe { - tokenize_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_ERROR_STRING_ALLOCATION_FAILED, - 0, - ptr::null_mut(), - ) - }; - - assert_eq!(result, Err(StringToTokenError::NotEnoughMemory)); - } - - #[test] - fn tokenize_cxx_exception_without_a_message_is_a_contract_error() { - let result = unsafe { - tokenize_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_LLAMA_CPP_THREW_CXX_EXCEPTION, - 0, - ptr::null_mut(), - ) - }; - - assert_eq!( - result, - Err(crate::FfiContractError { - operation: "llama_rs_tokenize", - detail: "reported a thrown C++ exception without an error message", - } - .into()) - ); - } - - #[test] - fn tokenize_unknown_status_is_preserved() { - let result = unsafe { tokenize_status_to_result(255, 0, ptr::null_mut()) }; - - assert_eq!( - result, - Err(StringToTokenError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_tokenize", - code: 255, - })) - ); - } - - #[test] - fn tokenize_status_to_result_maps_null_vocab_arg() { - let outcome = unsafe { - tokenize_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_VOCAB_ARG, - 0, - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_tokenize", - detail: "was given a null vocab argument", - } - .into() - ) - ); - } - - #[test] - fn tokenize_status_to_result_maps_null_text_arg() { - let outcome = unsafe { - tokenize_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_TEXT_ARG, - 0, - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_tokenize", - detail: "was given a null text argument", - } - .into() - ) - ); - } - - #[test] - fn tokenize_status_to_result_maps_null_out_returned_count_arg() { - let outcome = unsafe { - tokenize_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_OUT_RETURNED_COUNT_ARG, - 0, - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_tokenize", - detail: "was given a null out_returned_count argument", - } - .into() - ) - ); - } - - #[test] - fn tokenize_status_to_result_maps_null_out_error_arg() { - let outcome = unsafe { - tokenize_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_OUT_ERROR_ARG, - 0, - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_tokenize", - detail: "was given a null out_error argument", - } - .into() - ) - ); - } - - #[test] - fn tokenize_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = unsafe { - tokenize_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_LLAMA_CPP_OUT_OF_MEMORY, - 0, - ptr::null_mut(), - ) - }; - assert_eq!(outcome.err(), Some(StringToTokenError::LlamaCppOutOfMemory)); - } -} - -#[cfg(test)] -mod apply_chat_template_status_to_result_tests { - use std::ptr; - - use super::apply_chat_template_status_to_result; - use crate::error::apply_chat_template_error::ApplyChatTemplateError; - - #[test] - fn apply_chat_template_ok_returns_rendered_prompt() { - let rendered = std::ffi::CString::new("rendered prompt").unwrap(); - let out_string = unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(rendered.as_ptr()) }; - let result = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_OK, - out_string, - ptr::null_mut(), - ) - }; - - assert_eq!(result, Ok("rendered prompt".to_owned())); - } - - #[test] - fn apply_chat_template_ok_with_null_rendered_prompt_is_contract_error() { - let result = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_OK, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result, - Err(ApplyChatTemplateError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_apply_chat_template", - detail: "success status contained a null rendered string", - } - )) - ); - } - - #[test] - fn apply_chat_template_no_vocab_maps_to_no_vocab() { - let result = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_MODEL_HAS_NO_VOCAB, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!(result, Err(ApplyChatTemplateError::NoVocab)); - } - - #[test] - fn apply_chat_template_application_failed_maps_to_template_application_failed() { - let result = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_TEMPLATE_APPLICATION_FAILED, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result, - Err(ApplyChatTemplateError::TemplateApplicationFailed) - ); - } - - #[test] - fn apply_chat_template_allocation_failed_maps_to_not_enough_memory() { - let result = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!(result, Err(ApplyChatTemplateError::NotEnoughMemory)); - } - - #[test] - fn apply_chat_template_cxx_exception_is_reported() { - let message = std::ffi::CString::new("renderer exploded").unwrap(); - let out_error = unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(message.as_ptr()) }; - let result = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null_mut(), - out_error, ) }; assert_eq!( result, - Err(ApplyChatTemplateError::Reported { - message: "renderer exploded".to_owned() - }) + Err(crate::FfiContractError { + operation: "llama_rs_tokenize", + detail: "reported a thrown C++ exception without an error message", + } + .into()) ); } #[test] - fn apply_chat_template_unknown_status_is_preserved() { - let result = - unsafe { apply_chat_template_status_to_result(255, ptr::null_mut(), ptr::null_mut()) }; + fn tokenize_unknown_status_is_preserved() { + let result = unsafe { tokenize_status_to_result(255, 0, ptr::null_mut()) }; assert_eq!( result, - Err(ApplyChatTemplateError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_apply_chat_template", + Err(StringToTokenError::FfiStatus(crate::FfiStatusError { + operation: "llama_rs_tokenize", code: 255, })) ); } #[test] - fn apply_chat_template_status_to_result_maps_null_model_arg() { - let outcome = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_MODEL_ARG, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_apply_chat_template", - detail: "was given a null model argument", - } - .into() - ) - ); - } - - #[test] - fn apply_chat_template_status_to_result_maps_null_template_arg() { + fn tokenize_status_to_result_maps_null_vocab_arg() { let outcome = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_TEMPLATE_ARG, - ptr::null_mut(), + tokenize_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_VOCAB_ARG, + 0, ptr::null_mut(), ) }; @@ -3293,8 +1354,8 @@ mod apply_chat_template_status_to_result_tests { outcome.err(), Some( crate::FfiContractError { - operation: "llama_rs_apply_chat_template", - detail: "was given a null template argument", + operation: "llama_rs_tokenize", + detail: "was given a null vocab argument", } .into() ) @@ -3302,11 +1363,11 @@ mod apply_chat_template_status_to_result_tests { } #[test] - fn apply_chat_template_status_to_result_maps_null_messages_arg() { + fn tokenize_status_to_result_maps_null_text_arg() { let outcome = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_MESSAGES_ARG, - ptr::null_mut(), + tokenize_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_TEXT_ARG, + 0, ptr::null_mut(), ) }; @@ -3314,8 +1375,8 @@ mod apply_chat_template_status_to_result_tests { outcome.err(), Some( crate::FfiContractError { - operation: "llama_rs_apply_chat_template", - detail: "was given a null messages argument", + operation: "llama_rs_tokenize", + detail: "was given a null text argument", } .into() ) @@ -3323,11 +1384,11 @@ mod apply_chat_template_status_to_result_tests { } #[test] - fn apply_chat_template_status_to_result_maps_null_out_string_arg() { + fn tokenize_status_to_result_maps_null_out_returned_count_arg() { let outcome = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_OUT_STRING_ARG, - ptr::null_mut(), + tokenize_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_OUT_RETURNED_COUNT_ARG, + 0, ptr::null_mut(), ) }; @@ -3335,8 +1396,8 @@ mod apply_chat_template_status_to_result_tests { outcome.err(), Some( crate::FfiContractError { - operation: "llama_rs_apply_chat_template", - detail: "was given a null out_string argument", + operation: "llama_rs_tokenize", + detail: "was given a null out_returned_count argument", } .into() ) @@ -3344,11 +1405,11 @@ mod apply_chat_template_status_to_result_tests { } #[test] - fn apply_chat_template_status_to_result_maps_null_out_error_arg() { + fn tokenize_status_to_result_maps_null_out_error_arg() { let outcome = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_OUT_ERROR_ARG, - ptr::null_mut(), + tokenize_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_OUT_ERROR_ARG, + 0, ptr::null_mut(), ) }; @@ -3356,7 +1417,7 @@ mod apply_chat_template_status_to_result_tests { outcome.err(), Some( crate::FfiContractError { - operation: "llama_rs_apply_chat_template", + operation: "llama_rs_tokenize", detail: "was given a null out_error argument", } .into() @@ -3365,178 +1426,14 @@ mod apply_chat_template_status_to_result_tests { } #[test] - fn apply_chat_template_status_to_result_maps_llama_cpp_out_of_memory() { + fn tokenize_status_to_result_maps_llama_cpp_out_of_memory() { let outcome = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ApplyChatTemplateError::LlamaCppOutOfMemory) - ); - } -} - -#[cfg(test)] -mod chat_parser_free_status_to_result_tests { - use std::ptr; - - use super::chat_parser_free_status_to_result; - use crate::ParseChatMessageError; - - #[test] - fn chat_parser_free_ok_is_success() { - let result = unsafe { - chat_parser_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_FREE_OK, - ptr::null_mut(), - ) - }; - - assert!( - result.is_ok(), - "a clean destructor must not report a failure" - ); - } - - #[test] - fn chat_parser_free_allocation_failed_is_not_enough_memory() { - let result = unsafe { - chat_parser_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_FREE_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - ) - }; - - assert_eq!(result.err(), Some(ParseChatMessageError::NotEnoughMemory)); - } - - #[test] - fn chat_parser_free_llama_cpp_out_of_memory_is_preserved() { - let result = unsafe { - chat_parser_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_FREE_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null_mut(), - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::LlamaCppOutOfMemory) - ); - } - - #[test] - fn chat_parser_free_destructor_threw_surfaces_the_message() { - let out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"the destructor threw".as_ptr()) - }; - let result = unsafe { - chat_parser_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - out_error, - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::DestructorFailed { - message: "the destructor threw".to_owned() - }) - ); - } - - #[test] - fn chat_parser_free_unknown_status_is_preserved() { - let result = unsafe { chat_parser_free_status_to_result(255, ptr::null_mut()) }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_chat_parser_free", - code: 255, - })) - ); - } -} - -#[cfg(test)] -mod reasoning_markers_free_status_to_result_tests { - use std::ptr; - - use super::reasoning_markers_free_status_to_result; - use crate::MarkerDetectionError; - - #[test] - fn reasoning_markers_free_ok_is_success() { - let result = unsafe { - reasoning_markers_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_REASONING_MARKERS_FREE_OK, - ptr::null_mut(), - ) - }; - - assert_eq!(result, Ok(())); - } - - #[test] - fn reasoning_markers_free_allocation_failed_is_not_enough_memory() { - let result = unsafe { - reasoning_markers_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_REASONING_MARKERS_FREE_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - ) - }; - - assert_eq!(result, Err(MarkerDetectionError::NotEnoughMemory)); - } - - #[test] - fn reasoning_markers_free_llama_cpp_out_of_memory_is_preserved() { - let result = unsafe { - reasoning_markers_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_REASONING_MARKERS_FREE_LLAMA_CPP_OUT_OF_MEMORY, + tokenize_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_LLAMA_CPP_OUT_OF_MEMORY, + 0, ptr::null_mut(), ) }; - - assert_eq!(result, Err(MarkerDetectionError::LlamaCppOutOfMemory)); - } - - #[test] - fn reasoning_markers_free_destructor_threw_surfaces_the_message() { - let out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"the destructor threw".as_ptr()) - }; - let result = unsafe { - reasoning_markers_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_REASONING_MARKERS_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - out_error, - ) - }; - - assert_eq!( - result, - Err(MarkerDetectionError::ReasoningMarkersFreeFailed { - message: "the destructor threw".to_owned(), - }) - ); - } - - #[test] - fn reasoning_markers_free_unknown_status_is_preserved() { - let result = unsafe { reasoning_markers_free_status_to_result(255, ptr::null_mut()) }; - - assert_eq!( - result, - Err(crate::FfiStatusError { - operation: "llama_rs_reasoning_markers_free", - code: 255, - } - .into()) - ); + assert_eq!(outcome.err(), Some(StringToTokenError::LlamaCppOutOfMemory)); } } diff --git a/llama-cpp-bindings/src/model/params.rs b/llama-cpp-bindings/src/model/params.rs index fe742373c..ce8f46a29 100644 --- a/llama-cpp-bindings/src/model/params.rs +++ b/llama-cpp-bindings/src/model/params.rs @@ -502,6 +502,7 @@ mod tests { use super::LLAMA_CPP_MAX_DEVICES; use super::LlamaModelParams; use super::fit_params_status_to_result; + use crate::LlamaCppError; use crate::error::fit_error::FitError; use crate::model::llama_lazy_mode::LlamaLazyMode; use crate::model::llama_lazy_mode_parse_error::LlamaLazyModeParseError; @@ -713,16 +714,6 @@ mod tests { assert_eq!(params.unwrap().devices(), Vec::::new()); } - #[test] - fn with_devices_invalid_index_returns_error() { - let result = LlamaModelParams::default().with_devices(&[999_999]); - - assert_eq!( - std::mem::discriminant(&result.unwrap_err()), - std::mem::discriminant(&crate::LlamaCppError::BackendDeviceNotFound(0)), - ); - } - #[test] fn add_cpu_buft_override_succeeds() { let mut params = std::pin::pin!(LlamaModelParams::default()); @@ -782,12 +773,11 @@ mod tests { let too_many: Vec = (0..17).collect(); let result = LlamaModelParams::default().with_devices(&too_many); - assert!( - result - .unwrap_err() - .to_string() - .contains("Max devices exceeded") - ); + let Err(LlamaCppError::MaxDevicesExceeded(max_devices)) = result else { + panic!("17 devices must exceed the device limit"); + }; + + assert_eq!(max_devices, crate::max_devices().min(LLAMA_CPP_MAX_DEVICES)); } #[test] @@ -809,7 +799,11 @@ mod tests { let invalid_index = usize::MAX; let result = LlamaModelParams::default().with_devices(&[invalid_index]); - assert!(result.unwrap_err().to_string().contains("Backend device")); + let Err(LlamaCppError::BackendDeviceNotFound(device)) = result else { + panic!("an out-of-range device index must not be found"); + }; + + assert_eq!(device, invalid_index); } #[test] diff --git a/llama-cpp-bindings/src/parsed_chat.rs b/llama-cpp-bindings/src/parsed_chat.rs new file mode 100644 index 000000000..4cf066a75 --- /dev/null +++ b/llama-cpp-bindings/src/parsed_chat.rs @@ -0,0 +1,237 @@ +use std::ffi::c_char; +use std::ptr; +use std::ptr::NonNull; + +use llama_cpp_bindings_types::ModelProvidedToolCallId; +use llama_cpp_bindings_types::ModelProvidedToolCallIdError; +use llama_cpp_bindings_types::ParsedChatMessage; +use llama_cpp_bindings_types::ParsedToolCall; +use llama_cpp_bindings_types::ParsedToolCallId; +use llama_cpp_bindings_types::ToolCallArguments; +use llama_cpp_ffi_status::read_and_free_cpp_string; + +use crate::cpp_text_view::cpp_text_view; +use crate::error::parse_chat_message_error::ParseChatMessageError; +use crate::result_after_release::result_after_release; + +/// # Safety +/// +/// `free_error` must be the pointer populated by the preceding +/// `llama_rs_parsed_chat_free` call, or null. The destructor-threw arm reads and +/// frees it. +unsafe fn parsed_chat_free_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_parsed_chat_free_status, + free_error: *mut c_char, +) -> Result<(), ParseChatMessageError> { + match status { + llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_OK => Ok(()), + llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_ERROR_STRING_ALLOCATION_FAILED => { + Err(ParseChatMessageError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_LLAMA_CPP_OUT_OF_MEMORY => { + Err(ParseChatMessageError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + free_error, + "llama_rs_parsed_chat_free", + "reported a thrown C++ exception without an error message", + ) + }?; + + Err(ParseChatMessageError::DestructorFailed { message }) + } + other => Err(crate::FfiStatusError { + operation: "llama_rs_parsed_chat_free", + code: i64::from(other), + } + .into()), + } +} + +fn parsed_text( + read: impl FnOnce(*mut usize) -> *const c_char, +) -> Result { + let mut length = 0; + let text = read(&raw mut length); + + unsafe { cpp_text_view(text, length) } + .map(str::to_owned) + .map_err(ParseChatMessageError::ParsedTextNotUtf8) +} + +fn tool_call_id_reported_by_llama_cpp(llama_cpp_tool_call_id: String) -> ParsedToolCallId { + match ModelProvidedToolCallId::try_from(llama_cpp_tool_call_id) { + Ok(model_provided_tool_call_id) => { + ParsedToolCallId::ModelProvided(model_provided_tool_call_id) + } + Err(ModelProvidedToolCallIdError::Empty) => ParsedToolCallId::NotProvidedByModel, + } +} + +fn parsed_tool_call( + handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, + index: usize, +) -> Result { + let id = parsed_text(|out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_id(handle, index, out_length) + })?; + let name = parsed_text(|out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_name(handle, index, out_length) + })?; + let arguments = parsed_text(|out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_arguments(handle, index, out_length) + })?; + + Ok(ParsedToolCall::new( + tool_call_id_reported_by_llama_cpp(id), + name, + ToolCallArguments::from_string(arguments), + )) +} + +fn collect_parsed_chat_message( + handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, +) -> Result { + let content = parsed_text(|out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_parsed_chat_content(handle, out_length) + })?; + let reasoning_content = parsed_text(|out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_parsed_chat_reasoning_content(handle, out_length) + })?; + let tool_call_count = + unsafe { llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_count(handle) }; + let tool_calls = (0..tool_call_count) + .map(|index| parsed_tool_call(handle, index)) + .collect::, ParseChatMessageError>>()?; + + Ok(ParsedChatMessage::new( + content, + reasoning_content, + tool_calls, + )) +} + +/// Reads the message llama.cpp parsed into `handle` and releases the handle. +/// +/// # Errors +/// +/// Returns [`ParseChatMessageError`] when an accessor fails or reports text that is not UTF-8, +/// or when releasing the handle fails. +pub fn take_parsed_chat_message( + handle: NonNull, +) -> Result { + let message = collect_parsed_chat_message(handle.as_ptr()); + let mut free_error: *mut c_char = ptr::null_mut(); + let free_status = unsafe { + llama_cpp_bindings_sys::llama_rs_parsed_chat_free(handle.as_ptr(), &raw mut free_error) + }; + let freed = unsafe { parsed_chat_free_status_to_result(free_status, free_error) }; + + result_after_release(message, freed) +} + +#[cfg(test)] +mod tests { + use std::ptr; + + use llama_cpp_bindings_types::ModelProvidedToolCallId; + use llama_cpp_bindings_types::ParsedToolCallId; + + use super::parsed_chat_free_status_to_result; + use super::tool_call_id_reported_by_llama_cpp; + use crate::error::parse_chat_message_error::ParseChatMessageError; + + #[test] + fn an_empty_llama_cpp_tool_call_id_is_not_provided_by_the_model() { + assert_eq!( + tool_call_id_reported_by_llama_cpp(String::new()), + ParsedToolCallId::NotProvidedByModel + ); + } + + #[test] + fn a_non_empty_llama_cpp_tool_call_id_is_provided_by_the_model() { + assert_eq!( + Ok(tool_call_id_reported_by_llama_cpp("abc123def".to_owned())), + ModelProvidedToolCallId::try_from("abc123def".to_owned()) + .map(ParsedToolCallId::ModelProvided) + ); + } + + #[test] + fn parsed_chat_free_ok_is_success() { + let result = unsafe { + parsed_chat_free_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_OK, + ptr::null_mut(), + ) + }; + + assert!( + result.is_ok(), + "a clean destructor must not report a failure" + ); + } + + #[test] + fn parsed_chat_free_allocation_failed_is_not_enough_memory() { + let result = unsafe { + parsed_chat_free_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_ERROR_STRING_ALLOCATION_FAILED, + ptr::null_mut(), + ) + }; + + assert_eq!(result.err(), Some(ParseChatMessageError::NotEnoughMemory)); + } + + #[test] + fn parsed_chat_free_llama_cpp_out_of_memory_is_preserved() { + let result = unsafe { + parsed_chat_free_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_LLAMA_CPP_OUT_OF_MEMORY, + ptr::null_mut(), + ) + }; + + assert_eq!( + result.err(), + Some(ParseChatMessageError::LlamaCppOutOfMemory) + ); + } + + #[test] + fn parsed_chat_free_destructor_threw_surfaces_the_message() { + let out_error = unsafe { + llama_cpp_bindings_sys::llama_rs_string_dup(c"the destructor threw".as_ptr()) + }; + let result = unsafe { + parsed_chat_free_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, + out_error, + ) + }; + + assert_eq!( + result.err(), + Some(ParseChatMessageError::DestructorFailed { + message: "the destructor threw".to_owned() + }) + ); + } + + #[test] + fn parsed_chat_free_unknown_status_is_preserved() { + let result = unsafe { parsed_chat_free_status_to_result(255, ptr::null_mut()) }; + + assert_eq!( + result.err(), + Some(ParseChatMessageError::FfiStatus(crate::FfiStatusError { + operation: "llama_rs_parsed_chat_free", + code: 255, + })) + ); + } +} diff --git a/llama-cpp-bindings/src/raw_chat_message.rs b/llama-cpp-bindings/src/raw_chat_message.rs index 15b2f2662..1d2bfe4d6 100644 --- a/llama-cpp-bindings/src/raw_chat_message.rs +++ b/llama-cpp-bindings/src/raw_chat_message.rs @@ -1,24 +1,8 @@ +use crate::chat_output_completeness::ChatOutputCompleteness; + #[derive(Debug, Eq, PartialEq)] pub struct RawChatMessage { pub text: String, - pub is_partial: bool, + pub completeness: ChatOutputCompleteness, pub ffi_error_message: String, } - -#[cfg(test)] -mod tests { - use super::RawChatMessage; - - #[test] - fn carries_text_partial_flag_and_ffi_error_message() { - let raw = RawChatMessage { - text: "hello".to_owned(), - is_partial: true, - ffi_error_message: "parser bailed".to_owned(), - }; - - assert_eq!(raw.text, "hello"); - assert!(raw.is_partial); - assert_eq!(raw.ffi_error_message, "parser bailed"); - } -} diff --git a/llama-cpp-bindings/src/resolved_tool_call_markers.rs b/llama-cpp-bindings/src/resolved_tool_call_markers.rs deleted file mode 100644 index 3b2e6a3c4..000000000 --- a/llama-cpp-bindings/src/resolved_tool_call_markers.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct ResolvedToolCallMarkers { - pub open: Option, - pub close: Option, -} diff --git a/llama-cpp-bindings/src/result_after_release.rs b/llama-cpp-bindings/src/result_after_release.rs new file mode 100644 index 000000000..b99b8b72d --- /dev/null +++ b/llama-cpp-bindings/src/result_after_release.rs @@ -0,0 +1,50 @@ +use std::fmt::Display; + +/// Combines the result of reading a llama.cpp handle with the result of releasing it: a read +/// failure wins and a release failure that accompanies it is logged, since only one error can be +/// returned. +/// +/// # Errors +/// +/// Returns the read failure, or the release failure after a successful read. +pub fn result_after_release( + read: Result, + released: Result<(), TError>, +) -> Result { + match read { + Ok(value) => released.map(|()| value), + Err(read_failure) => { + if let Err(release_failure) = released { + log::error!("{release_failure}"); + } + + Err(read_failure) + } + } +} + +#[cfg(test)] +mod tests { + use super::result_after_release; + + #[test] + fn a_successful_read_and_release_yields_the_value() { + assert_eq!(result_after_release::(Ok(7), Ok(())), Ok(7)); + } + + #[test] + fn a_failed_release_after_a_successful_read_is_returned() { + assert_eq!( + result_after_release::(Ok(7), Err("release".to_owned())), + Err("release".to_owned()) + ); + } + + #[test] + fn a_failed_read_wins_over_a_failed_release() { + assert_eq!( + result_after_release::(Err("read".to_owned()), Err("release".to_owned())), + Err("read".to_owned()) + ); + } +} diff --git a/llama-cpp-bindings/src/sampled_token_classifier.rs b/llama-cpp-bindings/src/sampled_token_classifier.rs index 0a2b9879c..05f58720a 100644 --- a/llama-cpp-bindings/src/sampled_token_classifier.rs +++ b/llama-cpp-bindings/src/sampled_token_classifier.rs @@ -1,21 +1,24 @@ -use std::collections::VecDeque; -use std::sync::Arc; - use llama_cpp_bindings_sys::llama_pos; use llama_cpp_bindings_sys::llama_seq_id; use llama_cpp_bindings_types::TokenUsage; use llama_cpp_bindings_types::TokenUsageError; +use crate::bare_json_tool_call_probe::BareJsonToolCallProbe; use crate::bare_json_tool_calls::BareJsonToolCalls; use crate::batch_add_error::BatchAddError; +use crate::chat_classification::ChatClassification; +use crate::chat_output::ChatOutput; +use crate::chat_output_completeness::ChatOutputCompleteness; +use crate::chat_piece_role::ChatPieceRole; +use crate::chat_session::ChatSession; use crate::context::LlamaContext; use crate::error::EvalMultimodalChunksError; use crate::error::SampleError; -use crate::error::TokenToStringError; +use crate::error::ingest_error::IngestError; use crate::eval_multimodal_chunks_params::EvalMultimodalChunksParams; use crate::generation_progress::GenerationProgress; -use crate::json_probe_outcome::JsonProbeOutcome; +use crate::ingest_outcome::IngestOutcome; use crate::llama_batch::LlamaBatch; use crate::model::LlamaModel; use crate::mtmd::FittedMtmdInputChunk; @@ -26,331 +29,245 @@ use crate::mtmd::MtmdInputChunks; use crate::mtmd::mtmd_eval_limits::MtmdEvalLimits; use crate::sampled_token::SampledToken; use crate::sampling::LlamaSampler; -use crate::streaming_json_probe::StreamingJsonProbe; -use crate::streaming_markers::StreamingMarkers; use crate::token::LlamaToken; use crate::token_piece::TokenPiece; pub use crate::classified_sample::ClassifiedSample; -use crate::ingest_outcome::IngestOutcome; pub use crate::sampled_token_section::SampledTokenSection; -#[derive(Copy, Clone, Debug)] -enum PendingMarkerStatus { - Unmatched, - ResolvedBoundary, - AmbiguousBoundary, +enum ContentProbe { + Disabled, + BareJson(BareJsonToolCallProbe), } -#[derive(Clone, Debug)] -struct PendingToken { - token: LlamaToken, - decoded: String, - section: SampledTokenSection, - section_before_token: SampledTokenSection, - marker_status: PendingMarkerStatus, - is_from_prompt: bool, +impl ContentProbe { + fn pass(&mut self, outcome: IngestOutcome, released: &mut Vec) { + match self { + Self::Disabled => released.push(outcome), + Self::BareJson(probe) => probe.pass(outcome, released), + } + } + + fn release(&mut self, released: &mut Vec) { + match self { + Self::Disabled => {} + Self::BareJson(probe) => probe.release(released), + } + } +} + +struct ChatSessionAttribution<'session, 'model> { + session: &'session ChatSession<'model>, + preserved_tokens: Vec, + output: ChatOutput, + attributed_tokens: usize, + content_probe: ContentProbe, } -#[derive(Clone, Debug, Eq, PartialEq)] -enum ProbeMode { - Idle, - Active { - probe: StreamingJsonProbe, - held_count: usize, - }, +enum TokenAttribution<'session, 'model> { + Undeterminable, + ChatSession(ChatSessionAttribution<'session, 'model>), } -pub struct SampledTokenClassifier<'model> { +const fn sampled_token_in(section: SampledTokenSection, token: LlamaToken) -> SampledToken { + match section { + SampledTokenSection::Reasoning => SampledToken::Reasoning(token), + SampledTokenSection::Content => SampledToken::Content(token), + SampledTokenSection::ToolCall => SampledToken::ToolCall(token), + SampledTokenSection::Pending => SampledToken::Undeterminable(token), + } +} + +const fn section_of(sampled_token: SampledToken) -> SampledTokenSection { + match sampled_token { + SampledToken::Reasoning(_) => SampledTokenSection::Reasoning, + SampledToken::Content(_) => SampledTokenSection::Content, + SampledToken::ToolCall(_) => SampledTokenSection::ToolCall, + SampledToken::Undeterminable(_) => SampledTokenSection::Pending, + } +} + +const fn outcome_of(token: LlamaToken, role: ChatPieceRole, piece: String) -> IngestOutcome { + IngestOutcome { + sampled_token: sampled_token_in(role.section(), token), + piece: match role { + ChatPieceRole::Text(_) => TokenPiece::Visible(piece), + ChatPieceRole::Marker(_) => TokenPiece::Marker(piece), + }, + } +} + +/// Attributes every generated token to reasoning, content, or a tool call, the way llama.cpp's +/// parse of the chat template attributes the text the token decoded to, and counts the tokens +/// of each section. +pub struct SampledTokenClassifier<'session, 'model> { model: &'model LlamaModel, - markers: Arc, - marker_lookback: usize, - bare_json_tool_calls: BareJsonToolCalls, + attribution: TokenAttribution<'session, 'model>, decoder: encoding_rs::Decoder, - pending: VecDeque, section: SampledTokenSection, pending_prompt_tokens: u64, usage: TokenUsage, - probe_mode: ProbeMode, } -impl<'model> SampledTokenClassifier<'model> { +impl<'session, 'model> SampledTokenClassifier<'session, 'model> { + /// A classifier for generation outside of a chat template, whose tokens belong to no + /// section. #[must_use] - pub fn new( - model: &'model LlamaModel, - markers: Arc, + pub fn undeterminable(model: &'model LlamaModel) -> Self { + Self { + model, + attribution: TokenAttribution::Undeterminable, + decoder: encoding_rs::UTF_8.new_decoder(), + section: SampledTokenSection::Pending, + pending_prompt_tokens: 0, + usage: TokenUsage::new(), + } + } + + /// A classifier for the answer to `session`. When the session's template cannot render tool + /// calls and `bare_json_tool_calls` asks for it, content that is a bare JSON tool call object + /// is attributed to a tool call. + #[must_use] + pub fn for_chat_session( + session: &'session ChatSession<'model>, bare_json_tool_calls: BareJsonToolCalls, ) -> Self { + let content_probe = if !session.supports_tool_calls() + && bare_json_tool_calls == BareJsonToolCalls::Detect + { + ContentProbe::BareJson(BareJsonToolCallProbe::default()) + } else { + ContentProbe::Disabled + }; + Self { - model, - marker_lookback: markers.max_token_len().saturating_sub(1), - markers, - bare_json_tool_calls, + model: session.model(), + attribution: TokenAttribution::ChatSession(ChatSessionAttribution { + session, + preserved_tokens: session.preserved_tokens(), + output: ChatOutput::default(), + attributed_tokens: 0, + content_probe, + }), decoder: encoding_rs::UTF_8.new_decoder(), - pending: VecDeque::new(), section: SampledTokenSection::Pending, pending_prompt_tokens: 0, usage: TokenUsage::new(), - probe_mode: ProbeMode::Idle, } } - /// Classifies a generated token and appends every outcome it finalises to - /// `outcomes`. An end-of-generation token is counted in the current section - /// without being detokenised, and releases every token still held back. + /// Classifies a generated token and appends every outcome it settles to `outcomes`. An + /// end-of-generation token settles every token still pending and is counted in the section + /// generation ended in. /// /// # Errors - /// Returns [`TokenToStringError`] when the sampled token cannot be - /// detokenised. The failure is surfaced rather than substituting an empty - /// piece, so classification never silently drops generated text. + /// + /// Returns [`IngestError`] when the token cannot be detokenized or llama.cpp fails while + /// parsing the answer. pub fn ingest( &mut self, token: LlamaToken, outcomes: &mut Vec, - ) -> Result { + ) -> Result { if self.model.is_eog_token(&SampledToken::Content(token)) { + self.finish(outcomes)?; self.record_usage_in(self.section); - self.finish(outcomes); return Ok(GenerationProgress::Ended); } - let decoded = self.decode(token)?; - - if self.markers.is_empty() { - self.usage.record_undeterminable_token(); - outcomes.push(IngestOutcome { - sampled_token: SampledToken::Undeterminable(token), - piece: TokenPiece::Visible(decoded), - }); - - return Ok(GenerationProgress::Continues); - } - - self.pending.push_back(PendingToken { - token, - decoded, - section: self.section, - section_before_token: self.section, - marker_status: PendingMarkerStatus::Unmatched, - is_from_prompt: false, - }); - - self.try_consume_marker_at_tail(); - self.classify_pending_tail(outcomes); - self.drain_overflow(outcomes); - - Ok(GenerationProgress::Continues) - } - - fn classify_pending_tail(&mut self, outcomes: &mut Vec) { - let Some(tail) = self.pending.back() else { - return; - }; - let section_disengages_probe = self.section_disengages_probe(); - let probe_engages = - matches!(self.probe_mode, ProbeMode::Idle) && self.probe_engages_on(&tail.decoded); - - let probe_outcome = match &mut self.probe_mode { - ProbeMode::Active { .. } if section_disengages_probe => { - self.abandon_probe_held_before_tail(outcomes); - - return; - } - ProbeMode::Active { probe, held_count } => { - *held_count += 1; - - probe.feed(&tail.decoded) + let attribution = match &mut self.attribution { + TokenAttribution::Undeterminable => { + let piece = self.model.token_to_piece( + &SampledToken::Content(token), + &mut self.decoder, + true, + None, + )?; + self.usage.record_undeterminable_token(); + outcomes.push(IngestOutcome { + sampled_token: SampledToken::Undeterminable(token), + piece: TokenPiece::Visible(piece), + }); + + return Ok(GenerationProgress::Continues); } - ProbeMode::Idle if probe_engages => { - let mut probe = StreamingJsonProbe::default(); - let probe_outcome = probe.feed(&tail.decoded); - - self.probe_mode = ProbeMode::Active { - probe, - held_count: 1, - }; - - probe_outcome - } - ProbeMode::Idle => return, + TokenAttribution::ChatSession(attribution) => attribution, }; + let renders_special = attribution.preserved_tokens.contains(&token); + let piece = self.model.token_to_piece( + &SampledToken::Content(token), + &mut self.decoder, + renders_special, + None, + )?; - match probe_outcome { - JsonProbeOutcome::StillPossiblyValid => {} - JsonProbeOutcome::CompletedValid => self.commit_probe_as_tool_call(outcomes), - JsonProbeOutcome::Failed => self.abandon_probe(outcomes), - } - } - - const fn section_disengages_probe(&self) -> bool { - matches!( - self.section, - SampledTokenSection::ToolCall | SampledTokenSection::Reasoning - ) - } - - fn probe_engages_on(&self, piece: &str) -> bool { - self.bare_json_tool_calls == BareJsonToolCalls::Detect - && matches!( - self.section, - SampledTokenSection::Content | SampledTokenSection::Pending - ) - && piece.trim_start().starts_with('{') - } - - pub fn ingest_prompt_token(&mut self, token: LlamaToken) { - if self.markers.is_empty() { - return; - } - - if self.model.is_eog_token(&SampledToken::Content(token)) { - self.pending.clear(); - self.section = SampledTokenSection::Pending; - - return; - } - - self.pending.push_back(PendingToken { - token, - decoded: String::new(), - section: self.section, - section_before_token: self.section, - marker_status: PendingMarkerStatus::Unmatched, - is_from_prompt: true, - }); - - self.try_consume_marker_at_tail(); - self.drain_overflow(&mut Vec::new()); - } - - pub fn ingest_prompt_tokens(&mut self, tokens: &[LlamaToken]) { - if self.markers.is_empty() { - return; - } - for &token in tokens { - self.ingest_prompt_token(token); - } - } - - /// Releases every generated token still held back, for a generation that - /// stops without an end-of-generation token (a token limit or a stop request). - pub fn finish(&mut self, outcomes: &mut Vec) { - self.probe_mode = ProbeMode::Idle; - - while let Some(entry) = self.pending.pop_front() { - if !entry.is_from_prompt { - outcomes.push(self.finalize_entry(entry)); - } - } - } - - fn decode(&mut self, token: LlamaToken) -> Result { - self.model - .token_to_piece(&SampledToken::Content(token), &mut self.decoder, true, None) - } - - fn try_consume_marker_at_tail(&mut self) { - let Some(marker) = self - .markers - .longest_matching_suffix(&self.pending.iter().rev().map(|entry| entry.token)) - else { - return; - }; - let span_start = self.pending.len() - marker.tokens().len(); - let span_section = marker.span_section(self.pending[span_start].section_before_token); - let next_section = marker.next_section(); - let is_ambiguous_prefix = self.markers.is_prefix_of_longer_marker(marker.tokens()); + attribution.output.push(token, &piece); + self.attribute(ChatOutputCompleteness::Partial, outcomes)?; - self.mark_marker_span(span_start, span_section, next_section, is_ambiguous_prefix); + Ok(GenerationProgress::Continues) } - fn mark_marker_span( + fn attribute( &mut self, - span_start: usize, - span_section: SampledTokenSection, - next_section: SampledTokenSection, - is_ambiguous_prefix: bool, - ) { - let marker_status = if is_ambiguous_prefix { - PendingMarkerStatus::AmbiguousBoundary - } else { - PendingMarkerStatus::ResolvedBoundary + completeness: ChatOutputCompleteness, + outcomes: &mut Vec, + ) -> Result<(), IngestError> { + let TokenAttribution::ChatSession(attribution) = &mut self.attribution else { + return Ok(()); }; - for entry in self.pending.iter_mut().skip(span_start) { - entry.marker_status = marker_status; - entry.section = span_section; - } - - self.section = next_section; - } - - const fn probe_held_count(&self) -> usize { - match self.probe_mode { - ProbeMode::Active { held_count, .. } => held_count, - ProbeMode::Idle => 0, - } - } - - fn drain_overflow(&mut self, outcomes: &mut Vec) { - let probe_held_count = self.probe_held_count(); - - while let Some(front) = self.pending.front() { - let drainable = self.pending.len().saturating_sub(probe_held_count); - - if drainable == 0 { - break; - } - - let beyond_lookback = drainable > self.marker_lookback; - let resolved_boundary = - matches!(front.marker_status, PendingMarkerStatus::ResolvedBoundary); - - if !resolved_boundary && !beyond_lookback { - break; - } - - let Some(entry) = self.pending.pop_front() else { + let classification = attribution + .session + .classify(&attribution.output, completeness)?; + let mut released = Vec::new(); + + while let Some(output_token) = attribution + .output + .tokens() + .get(attribution.attributed_tokens) + { + let role = match (&classification, completeness) { + (ChatClassification::Spans(spans), _) => spans.classify_token(&output_token.bytes), + (ChatClassification::OutputDoesNotMatchFormat, ChatOutputCompleteness::Partial) => { + None + } + ( + ChatClassification::OutputDoesNotMatchFormat, + ChatOutputCompleteness::Finished, + ) => Some(ChatPieceRole::Text(SampledTokenSection::Pending)), + }; + let Some(role) = role else { break; }; + let piece = attribution.output.text()[output_token.bytes.clone()].to_owned(); - if !entry.is_from_prompt { - outcomes.push(self.finalize_entry(entry)); - } + attribution + .content_probe + .pass(outcome_of(output_token.token, role, piece), &mut released); + attribution.attributed_tokens += 1; } - } - fn take_probe_held_tokens(&mut self) -> VecDeque { - let probe_held_count = self.probe_held_count(); - - self.probe_mode = ProbeMode::Idle; - - self.pending - .split_off(self.pending.len().saturating_sub(probe_held_count)) - } - - fn commit_probe_as_tool_call(&mut self, outcomes: &mut Vec) { - self.section = SampledTokenSection::Content; - - for mut entry in self.take_probe_held_tokens() { - entry.section = SampledTokenSection::ToolCall; - outcomes.push(self.finalize_entry(entry)); + if completeness == ChatOutputCompleteness::Finished { + attribution.content_probe.release(&mut released); } - } - fn abandon_probe(&mut self, outcomes: &mut Vec) { - for entry in self.take_probe_held_tokens() { - outcomes.push(self.finalize_entry(entry)); + for outcome in released { + self.section = section_of(outcome.sampled_token); + self.record_usage_in(self.section); + outcomes.push(outcome); } - } - fn abandon_probe_held_before_tail(&mut self, outcomes: &mut Vec) { - let Some(tail) = self.pending.pop_back() else { - return; - }; + Ok(()) + } - self.abandon_probe(outcomes); - self.pending.push_back(tail); + /// Settles every generated token still pending, for a generation that stops without an + /// end-of-generation token (a token limit or a stop request). + /// + /// # Errors + /// + /// Returns [`IngestError`] when llama.cpp fails while parsing the finished answer. + pub fn finish(&mut self, outcomes: &mut Vec) -> Result<(), IngestError> { + self.attribute(ChatOutputCompleteness::Finished, outcomes) } const fn record_usage_in(&mut self, section: SampledTokenSection) { @@ -362,34 +279,10 @@ impl<'model> SampledTokenClassifier<'model> { } } - fn finalize_entry(&mut self, entry: PendingToken) -> IngestOutcome { - self.record_usage_in(entry.section); - - let sampled_token = match entry.section { - SampledTokenSection::Reasoning => SampledToken::Reasoning(entry.token), - SampledTokenSection::Content => SampledToken::Content(entry.token), - SampledTokenSection::ToolCall => SampledToken::ToolCall(entry.token), - SampledTokenSection::Pending => SampledToken::Undeterminable(entry.token), - }; - - let piece = match entry.marker_status { - PendingMarkerStatus::Unmatched => TokenPiece::Visible(entry.decoded), - PendingMarkerStatus::ResolvedBoundary | PendingMarkerStatus::AmbiguousBoundary => { - TokenPiece::Marker(entry.decoded) - } - }; - - IngestOutcome { - sampled_token, - piece, - } - } - - /// Samples a token and classifies it, appending the outcomes it finalises - /// to `outcomes`. + /// Samples a token and classifies it, appending the outcomes it settles to `outcomes`. /// /// # Errors - /// Forwards [`LlamaSampler::sample`] errors verbatim. Nothing is recorded on failure. + /// Forwards [`LlamaSampler::sample`] and [`Self::ingest`] errors verbatim. pub fn sample( &mut self, sampler: &mut LlamaSampler, @@ -414,7 +307,6 @@ impl<'model> SampledTokenClassifier<'model> { logits: bool, ) -> Result<(), BatchAddError> { batch.add(&SampledToken::Content(token), position, seq_ids, logits)?; - self.ingest_prompt_token(token); self.pending_prompt_tokens = self.pending_prompt_tokens.saturating_add(1); Ok(()) @@ -430,7 +322,6 @@ impl<'model> SampledTokenClassifier<'model> { logits_all: bool, ) -> Result<(), BatchAddError> { batch.add_sequence(tokens, seq_id, logits_all)?; - self.ingest_prompt_tokens(tokens); self.pending_prompt_tokens = self .pending_prompt_tokens .saturating_add(tokens.len() as u64); @@ -508,10 +399,7 @@ impl<'model> SampledTokenClassifier<'model> { let n_tokens = fitted_chunk.chunk().n_tokens() as u64; match fitted_chunk.chunk_type() { - MtmdInputChunkType::Text => { - self.record_prompt_tokens(n_tokens); - self.ingest_prompt_tokens(fitted_chunk.text_tokens()); - } + MtmdInputChunkType::Text => self.record_prompt_tokens(n_tokens), MtmdInputChunkType::Image => self.record_input_image_tokens(n_tokens), MtmdInputChunkType::Audio => self.record_input_audio_tokens(n_tokens), } @@ -550,1184 +438,4 @@ impl<'model> SampledTokenClassifier<'model> { pub const fn current_section(&self) -> SampledTokenSection { self.section } - - #[must_use] - pub fn markers(&self) -> &StreamingMarkers { - &self.markers - } -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use super::PendingMarkerStatus; - use super::PendingToken; - use super::ProbeMode; - use super::SampledTokenClassifier; - use crate::bare_json_tool_calls::BareJsonToolCalls; - use crate::ingest_outcome::IngestOutcome; - use crate::marker_role::MarkerRole; - use crate::marker_role_candidate::MarkerRoleCandidate; - use crate::sampled_token::SampledToken; - use crate::sampled_token_section::SampledTokenSection; - use crate::streaming_json_probe::StreamingJsonProbe; - use crate::streaming_markers::StreamingMarkers; - use crate::token::LlamaToken; - - fn token(id: i32) -> LlamaToken { - LlamaToken::new(id) - } - - fn markers_with( - reasoning_open: Option>, - reasoning_close: Option>, - ) -> StreamingMarkers { - let candidates = reasoning_open - .into_iter() - .map(|tokens| MarkerRoleCandidate { - tokens, - role: MarkerRole::ReasoningOpen, - }) - .chain( - reasoning_close - .into_iter() - .map(|tokens| MarkerRoleCandidate { - tokens, - role: MarkerRole::ReasoningClose, - }), - ); - - StreamingMarkers::from_candidates(candidates).expect("synthetic markers must be valid") - } - - fn synthetic_classifier(markers: StreamingMarkers) -> SampledTokenClassifier<'static> { - SampledTokenClassifier { - model: unsafe { &*std::ptr::NonNull::::dangling().as_ptr() }, - marker_lookback: markers.max_token_len().saturating_sub(1), - bare_json_tool_calls: BareJsonToolCalls::Detect, - markers: Arc::new(markers), - decoder: encoding_rs::UTF_8.new_decoder(), - pending: std::collections::VecDeque::new(), - section: SampledTokenSection::Pending, - pending_prompt_tokens: 0, - usage: llama_cpp_bindings_types::TokenUsage::new(), - probe_mode: ProbeMode::Idle, - } - } - - fn push_pending(classifier: &mut SampledTokenClassifier<'_>, token_id: i32, decoded: &str) { - classifier.pending.push_back(PendingToken { - token: token(token_id), - decoded: decoded.to_owned(), - section: classifier.section, - section_before_token: classifier.section, - marker_status: PendingMarkerStatus::Unmatched, - is_from_prompt: false, - }); - } - - fn push_pending_from_prompt(classifier: &mut SampledTokenClassifier<'_>, token_id: i32) { - classifier.pending.push_back(PendingToken { - token: token(token_id), - decoded: String::new(), - section: classifier.section, - section_before_token: classifier.section, - marker_status: PendingMarkerStatus::Unmatched, - is_from_prompt: true, - }); - } - - fn push_and_probe( - classifier: &mut SampledTokenClassifier<'_>, - token_id: i32, - decoded: &str, - ) -> Vec { - push_pending(classifier, token_id, decoded); - classifier.try_consume_marker_at_tail(); - let mut outcomes = Vec::new(); - classifier.classify_pending_tail(&mut outcomes); - classifier.drain_overflow(&mut outcomes); - outcomes - } - - fn drained(classifier: &mut SampledTokenClassifier<'_>) -> Vec { - let mut outcomes = Vec::new(); - classifier.drain_overflow(&mut outcomes); - outcomes - } - - fn finished(classifier: &mut SampledTokenClassifier<'_>) -> Vec { - let mut outcomes = Vec::new(); - classifier.finish(&mut outcomes); - outcomes - } - - fn outcome_pieces(outcomes: &[IngestOutcome]) -> Vec<&str> { - outcomes - .iter() - .map(|outcome| outcome.piece.visible()) - .collect() - } - - fn outcome_sections(outcomes: &[IngestOutcome]) -> Vec { - outcomes - .iter() - .map(|outcome| match outcome.sampled_token { - SampledToken::Reasoning(_) => SampledTokenSection::Reasoning, - SampledToken::Content(_) => SampledTokenSection::Content, - SampledToken::ToolCall(_) => SampledTokenSection::ToolCall, - SampledToken::Undeterminable(_) => SampledTokenSection::Pending, - }) - .collect() - } - - fn markers_sharing_tool_call_open_with_a_reasoning_close( - shared: Vec, - ) -> StreamingMarkers { - StreamingMarkers::from_candidates([ - MarkerRoleCandidate { - tokens: vec![token(100)], - role: MarkerRole::ReasoningOpen, - }, - MarkerRoleCandidate { - tokens: vec![token(200)], - role: MarkerRole::ReasoningClose, - }, - MarkerRoleCandidate { - tokens: shared.clone(), - role: MarkerRole::ReasoningClose, - }, - MarkerRoleCandidate { - tokens: shared, - role: MarkerRole::ToolCallOpen, - }, - MarkerRoleCandidate { - tokens: vec![token(201)], - role: MarkerRole::ToolCallClose, - }, - ]) - .expect("synthetic markers must be valid") - } - - #[test] - fn a_token_that_is_both_a_reasoning_close_and_the_tool_call_open_opens_the_tool_call() { - let mut classifier = - synthetic_classifier(markers_sharing_tool_call_open_with_a_reasoning_close(vec![ - token(300), - ])); - classifier.section = SampledTokenSection::Content; - - push_pending(&mut classifier, 300, ""); - classifier.try_consume_marker_at_tail(); - - assert_eq!(classifier.section, SampledTokenSection::ToolCall); - } - - #[test] - fn a_shared_marker_ends_reasoning_by_opening_the_tool_call() { - let mut classifier = - synthetic_classifier(markers_sharing_tool_call_open_with_a_reasoning_close(vec![ - token(300), - ])); - classifier.section = SampledTokenSection::Reasoning; - - push_pending(&mut classifier, 300, ""); - classifier.try_consume_marker_at_tail(); - - assert_eq!(classifier.section, SampledTokenSection::ToolCall); - } - - #[test] - fn longer_marker_reclassifies_a_completed_marker_that_was_its_prefix() { - let markers = StreamingMarkers::from_candidates([ - MarkerRoleCandidate { - tokens: vec![token(300)], - role: MarkerRole::ReasoningClose, - }, - MarkerRoleCandidate { - tokens: vec![token(300), token(301)], - role: MarkerRole::ToolCallOpen, - }, - ]) - .expect("synthetic markers must be valid"); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Reasoning; - - push_pending(&mut classifier, 300, "::new()); - - push_pending(&mut classifier, 301, "_call>"); - classifier.try_consume_marker_at_tail(); - let outcomes = drained(&mut classifier); - - assert_eq!(classifier.section, SampledTokenSection::ToolCall); - assert_eq!( - outcome_sections(&outcomes), - vec![SampledTokenSection::ToolCall, SampledTokenSection::ToolCall] - ); - assert_eq!(outcome_pieces(&outcomes), vec!["", ""]); - } - - #[test] - fn longer_closing_marker_uses_section_from_before_its_shorter_prefix() { - let markers = StreamingMarkers::from_candidates([ - MarkerRoleCandidate { - tokens: vec![token(300)], - role: MarkerRole::ReasoningClose, - }, - MarkerRoleCandidate { - tokens: vec![token(300), token(301)], - role: MarkerRole::ReasoningClose, - }, - ]); - assert!(markers.is_ok()); - let markers = markers.unwrap_or_default(); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Reasoning; - - push_pending(&mut classifier, 300, "::new()); - - push_pending(&mut classifier, 301, ">"); - classifier.try_consume_marker_at_tail(); - let outcomes = drained(&mut classifier); - - assert_eq!(classifier.section, SampledTokenSection::Content); - assert_eq!( - outcome_sections(&outcomes), - vec![ - SampledTokenSection::Reasoning, - SampledTokenSection::Reasoning - ] - ); - assert_eq!(outcome_pieces(&outcomes), vec!["", ""]); - assert_eq!(classifier.usage().reasoning_tokens, 2); - assert_eq!(classifier.usage().content_tokens, 0); - assert_eq!(classifier.usage().tool_call_tokens, 0); - assert_eq!(classifier.usage().undeterminable_tokens, 0); - } - - #[test] - fn single_token_close_marker_when_already_in_reasoning_emits_empty_piece_for_marker() { - let markers = markers_with(Some(vec![token(100)]), Some(vec![token(200)])); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Reasoning; - - push_pending(&mut classifier, 7, "step"); - classifier.try_consume_marker_at_tail(); - let mut outcomes = drained(&mut classifier); - - push_pending(&mut classifier, 200, ""); - classifier.try_consume_marker_at_tail(); - outcomes.extend(drained(&mut classifier)); - - push_pending(&mut classifier, 9, "Hi"); - classifier.try_consume_marker_at_tail(); - outcomes.extend(drained(&mut classifier)); - - outcomes.extend(finished(&mut classifier)); - - assert_eq!( - outcome_sections(&outcomes), - vec![ - SampledTokenSection::Reasoning, - SampledTokenSection::Reasoning, - SampledTokenSection::Content, - ], - ); - assert_eq!(outcome_pieces(&outcomes), vec!["step", "", "Hi"]); - assert_eq!(classifier.section, SampledTokenSection::Content); - } - - #[test] - fn multi_token_close_marker_suppresses_every_marker_token() { - let markers = markers_with( - Some(vec![token(100)]), - Some(vec![token(200), token(201), token(202)]), - ); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Reasoning; - - let mut outcomes = Vec::new(); - for (id, decoded) in [(7, "r"), (200, ""), (9, "OK")] { - push_pending(&mut classifier, id, decoded); - classifier.try_consume_marker_at_tail(); - outcomes.extend(drained(&mut classifier)); - } - outcomes.extend(finished(&mut classifier)); - - assert_eq!(outcome_pieces(&outcomes), vec!["r", "", "", "", "OK"]); - assert_eq!(classifier.section, SampledTokenSection::Content); - } - - #[test] - fn marker_prefix_that_diverges_does_not_suppress_buffered_tokens() { - let markers = markers_with( - Some(vec![token(100)]), - Some(vec![token(200), token(201), token(202)]), - ); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Reasoning; - - let mut outcomes = Vec::new(); - for (id, decoded) in [(7, "r"), (200, "a"), (201, "b"), (300, "x")] { - push_pending(&mut classifier, id, decoded); - classifier.try_consume_marker_at_tail(); - outcomes.extend(drained(&mut classifier)); - } - outcomes.extend(finished(&mut classifier)); - - assert_eq!(outcome_pieces(&outcomes), vec!["r", "a", "b", "x"]); - assert!(outcomes.iter().all(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::Reasoning(LlamaToken::new(0))) - })); - assert_eq!(classifier.section, SampledTokenSection::Reasoning); - } - - #[test] - fn open_then_close_back_to_back_emits_two_empty_pieces_around_zero_content() { - let markers = markers_with(Some(vec![token(100)]), Some(vec![token(200)])); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let mut outcomes = Vec::new(); - for (id, decoded) in [(100, ""), (200, ""), (9, "Hi")] { - push_pending(&mut classifier, id, decoded); - classifier.try_consume_marker_at_tail(); - outcomes.extend(drained(&mut classifier)); - } - outcomes.extend(finished(&mut classifier)); - - assert_eq!( - outcome_sections(&outcomes), - vec![ - SampledTokenSection::Reasoning, - SampledTokenSection::Reasoning, - SampledTokenSection::Content, - ], - ); - assert_eq!(outcome_pieces(&outcomes), vec!["", "", "Hi"]); - assert_eq!(classifier.section, SampledTokenSection::Content); - } - - #[test] - fn spurious_reasoning_close_in_content_section_classifies_as_content() { - let markers = markers_with(Some(vec![token(100)]), Some(vec![token(200)])); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - push_pending(&mut classifier, 200, ""); - classifier.try_consume_marker_at_tail(); - let outcomes = drained(&mut classifier); - - assert_eq!( - outcome_sections(&outcomes), - vec![SampledTokenSection::Content], - ); - assert_eq!(classifier.section, SampledTokenSection::Content); - } - - #[test] - fn spurious_tool_call_close_in_reasoning_section_classifies_as_tool_call() { - let markers = StreamingMarkers::from_candidates([ - MarkerRoleCandidate { - tokens: vec![token(100)], - role: MarkerRole::ReasoningOpen, - }, - MarkerRoleCandidate { - tokens: vec![token(200)], - role: MarkerRole::ReasoningClose, - }, - MarkerRoleCandidate { - tokens: vec![token(300)], - role: MarkerRole::ToolCallOpen, - }, - MarkerRoleCandidate { - tokens: vec![token(400)], - role: MarkerRole::ToolCallClose, - }, - ]) - .expect("synthetic markers must be valid"); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::ToolCall; - - push_pending(&mut classifier, 400, ""); - classifier.try_consume_marker_at_tail(); - let outcomes = drained(&mut classifier); - - assert_eq!( - outcome_sections(&outcomes), - vec![SampledTokenSection::ToolCall], - ); - assert_eq!(classifier.section, SampledTokenSection::Content); - } - - #[test] - fn finish_releases_every_held_generated_token() { - let markers = markers_with( - Some(vec![token(100)]), - Some(vec![token(200), token(201), token(202)]), - ); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Reasoning; - - push_pending(&mut classifier, 7, "abc"); - push_pending(&mut classifier, 200, "::new()); - - assert_eq!(classifier.usage().reasoning_tokens, 0); - assert_eq!(classifier.usage().content_tokens, 0); - assert_eq!(classifier.usage().tool_call_tokens, 0); - assert_eq!(classifier.usage().undeterminable_tokens, 0); - } - - #[test] - fn prompt_token_completing_marker_with_generated_token_is_suppressed_correctly() { - let markers = markers_with( - Some(vec![token(100)]), - Some(vec![token(200), token(201), token(202)]), - ); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Reasoning; - - for token_id in [200, 201] { - push_pending_from_prompt(&mut classifier, token_id); - classifier.try_consume_marker_at_tail(); - drained(&mut classifier); - } - - assert_eq!(classifier.section, SampledTokenSection::Reasoning); - assert_eq!(classifier.pending.len(), 2); - - classifier.pending.push_back(PendingToken { - token: token(202), - decoded: "k>".to_owned(), - section: classifier.section, - section_before_token: classifier.section, - marker_status: PendingMarkerStatus::Unmatched, - is_from_prompt: false, - }); - classifier.try_consume_marker_at_tail(); - let outcomes = drained(&mut classifier); - - assert_eq!(outcomes.len(), 1); - assert_eq!( - std::mem::discriminant(&outcomes[0].sampled_token), - std::mem::discriminant(&SampledToken::Reasoning(LlamaToken::new(0))) - ); - assert_eq!(outcomes[0].piece.visible(), ""); - assert_eq!(outcomes[0].piece.raw(), "k>"); - - assert_eq!(classifier.section, SampledTokenSection::Content); - assert_eq!(classifier.usage().reasoning_tokens, 1); - assert_eq!(classifier.usage().content_tokens, 0); - } - - #[test] - fn ingest_prompt_tokens_with_multiple_round_trips_ends_in_content() { - let markers = markers_with(Some(vec![token(100)]), Some(vec![token(200)])); - let mut classifier = synthetic_classifier(markers); - - for token_id in [100, 7, 200, 100, 8, 200] { - push_pending_from_prompt(&mut classifier, token_id); - classifier.try_consume_marker_at_tail(); - drained(&mut classifier); - } - - assert_eq!(classifier.section, SampledTokenSection::Content); - assert_eq!(classifier.usage().reasoning_tokens, 0); - assert_eq!(classifier.usage().content_tokens, 0); - assert_eq!(classifier.usage().tool_call_tokens, 0); - assert_eq!(classifier.usage().undeterminable_tokens, 0); - } - - #[test] - fn ingest_prompt_tokens_initial_section_is_always_pending() { - let markers = markers_with(Some(vec![token(100)]), Some(vec![token(200)])); - let classifier = synthetic_classifier(markers); - - assert_eq!(classifier.section, SampledTokenSection::Pending); - } - - #[test] - fn ingest_prompt_tokens_then_drain_for_generated_token_classifies_correctly() { - let markers = markers_with(Some(vec![token(100)]), Some(vec![token(200)])); - let mut classifier = synthetic_classifier(markers); - - for token_id in [100, 7, 200] { - push_pending_from_prompt(&mut classifier, token_id); - classifier.try_consume_marker_at_tail(); - drained(&mut classifier); - } - - assert_eq!(classifier.section, SampledTokenSection::Content); - assert_eq!(classifier.usage().reasoning_tokens, 0); - assert_eq!(classifier.usage().content_tokens, 0); - - classifier.pending.push_back(PendingToken { - token: token(50), - decoded: "hi".to_owned(), - section: classifier.section, - section_before_token: classifier.section, - marker_status: PendingMarkerStatus::Unmatched, - is_from_prompt: false, - }); - classifier.try_consume_marker_at_tail(); - let outcomes = drained(&mut classifier); - - assert_eq!(outcomes.len(), 1); - assert_eq!( - std::mem::discriminant(&outcomes[0].sampled_token), - std::mem::discriminant(&SampledToken::Content(LlamaToken::new(0))) - ); - assert_eq!(outcomes[0].piece.visible(), "hi"); - assert_eq!(classifier.usage().content_tokens, 1); - assert_eq!(classifier.usage().reasoning_tokens, 0); - assert_eq!(classifier.usage().undeterminable_tokens, 0); - } - - #[test] - fn close_marker_in_content_section_is_suppressed_as_boundary() { - let markers = markers_with(Some(vec![token(100)]), Some(vec![token(200)])); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let mut outcomes = Vec::new(); - for (id, decoded) in [(7, "hi"), (200, ""), (8, "ok")] { - push_pending(&mut classifier, id, decoded); - classifier.try_consume_marker_at_tail(); - outcomes.extend(drained(&mut classifier)); - } - outcomes.extend(finished(&mut classifier)); - - assert_eq!( - outcome_sections(&outcomes), - vec![ - SampledTokenSection::Content, - SampledTokenSection::Content, - SampledTokenSection::Content, - ], - ); - assert_eq!(outcome_pieces(&outcomes), vec!["hi", "", "ok"]); - assert_eq!(classifier.section, SampledTokenSection::Content); - } - - #[test] - fn open_marker_in_reasoning_section_is_suppressed_as_boundary() { - let markers = markers_with(Some(vec![token(100)]), Some(vec![token(200)])); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Reasoning; - - let mut outcomes = Vec::new(); - for (id, decoded) in [(7, "step1"), (100, ""), (8, "step2")] { - push_pending(&mut classifier, id, decoded); - classifier.try_consume_marker_at_tail(); - outcomes.extend(drained(&mut classifier)); - } - outcomes.extend(finished(&mut classifier)); - - assert_eq!(outcome_pieces(&outcomes), vec!["step1", "", "step2"]); - assert_eq!(classifier.section, SampledTokenSection::Reasoning); - } - - #[test] - fn record_prompt_tokens_updates_usage() { - let markers = markers_with(None, None); - let mut classifier = synthetic_classifier(markers); - - classifier.record_prompt_tokens(7); - - assert_eq!(classifier.usage().prompt_tokens, 7); - } - - #[test] - fn record_cached_prompt_tokens_updates_usage_when_under_limit() { - let markers = markers_with(None, None); - let mut classifier = synthetic_classifier(markers); - classifier.record_prompt_tokens(10); - - classifier.record_cached_prompt_tokens(3).unwrap(); - - assert_eq!(classifier.usage().cached_prompt_tokens, 3); - } - - #[test] - fn record_cached_prompt_tokens_returns_error_when_over_prompt_total() { - let markers = markers_with(None, None); - let mut classifier = synthetic_classifier(markers); - classifier.record_prompt_tokens(2); - - let result = classifier.record_cached_prompt_tokens(5); - - assert!(result.is_err()); - } - - #[test] - fn markers_accessor_returns_configured_markers() { - let configured = markers_with(Some(vec![token(1)]), Some(vec![token(2)])); - let classifier = synthetic_classifier(configured); - - let returned = classifier.markers(); - - assert!(returned.iter().any(|marker| { - marker.tokens() == [token(1)] && marker.roles() == [MarkerRole::ReasoningOpen] - })); - assert!(returned.iter().any(|marker| { - marker.tokens() == [token(2)] && marker.roles() == [MarkerRole::ReasoningClose] - })); - } - - #[test] - fn into_usage_consumes_classifier_and_yields_usage_snapshot() { - let markers = markers_with(None, None); - let mut classifier = synthetic_classifier(markers); - classifier.record_prompt_tokens(11); - - let usage = classifier.into_usage(); - - assert_eq!(usage.prompt_tokens, 11); - } - - #[test] - fn spurious_tool_call_close_in_content_section_classifies_as_content() { - let markers = StreamingMarkers::from_candidates([MarkerRoleCandidate { - tokens: vec![token(300)], - role: MarkerRole::ToolCallClose, - }]) - .expect("synthetic markers must be valid"); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - push_pending(&mut classifier, 300, ""); - classifier.try_consume_marker_at_tail(); - let outcomes = drained(&mut classifier); - - assert_eq!( - outcome_sections(&outcomes), - vec![SampledTokenSection::Content], - ); - assert_eq!(classifier.section, SampledTokenSection::Content); - } - - fn markers_with_tool_call_open(tool_call_open: Vec) -> StreamingMarkers { - StreamingMarkers::from_candidates([MarkerRoleCandidate { - tokens: tool_call_open, - role: MarkerRole::ToolCallOpen, - }]) - .expect("synthetic markers must be valid") - } - - fn feed_json_string( - classifier: &mut SampledTokenClassifier<'_>, - text: &str, - starting_token_id: i32, - ) -> Vec { - let mut outcomes = Vec::new(); - for (offset, ch) in text.char_indices() { - let token_id = starting_token_id + i32::try_from(offset).unwrap_or(i32::MAX); - let mut buffer = [0_u8; 4]; - let chunk = ch.encode_utf8(&mut buffer); - outcomes.extend(push_and_probe(classifier, token_id, chunk)); - } - outcomes - } - - #[test] - fn json_probe_engages_when_first_non_whitespace_is_open_brace_in_content() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - push_and_probe(&mut classifier, 1, "{"); - - assert_ne!(classifier.probe_mode, ProbeMode::Idle); - } - - #[test] - fn json_probe_releases_tokens_as_tool_call_when_signature_matches() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string(&mut classifier, r#"{"name":"f","arguments":{}}"#, 100); - - assert_ne!(outcomes, Vec::::new()); - let sections = outcome_sections(&outcomes); - assert!( - sections - .iter() - .all(|section| *section == SampledTokenSection::ToolCall), - "every emitted outcome should be ToolCall, got {sections:?}", - ); - assert_eq!(classifier.probe_mode, ProbeMode::Idle); - } - - #[test] - fn json_probe_releases_tokens_as_content_when_signature_does_not_match() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string(&mut classifier, r#"{"foo":"bar"}"#, 100); - - let sections = outcome_sections(&outcomes); - assert!( - sections - .iter() - .all(|section| *section == SampledTokenSection::Content), - "every emitted outcome should be Content, got {sections:?}", - ); - assert_eq!(classifier.probe_mode, ProbeMode::Idle); - } - - #[test] - fn json_probe_releases_tokens_as_content_when_extra_top_level_key() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string( - &mut classifier, - r#"{"name":"f","arguments":{},"extra":1}"#, - 100, - ); - - assert!(outcomes.iter().all(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::Content(LlamaToken::new(0))) - })); - } - - #[test] - fn json_probe_releases_tokens_as_content_when_arguments_is_not_object() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string(&mut classifier, r#"{"name":"f","arguments":"hi"}"#, 100); - - assert!(outcomes.iter().all(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::Content(LlamaToken::new(0))) - })); - } - - #[test] - fn json_probe_handles_strings_with_quoted_braces_in_arguments() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string( - &mut classifier, - r#"{"name":"f","arguments":{"q":"a } b"}}"#, - 100, - ); - - assert!(outcomes.iter().all(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::ToolCall(LlamaToken::new(0))) - })); - } - - #[test] - fn json_probe_handles_escaped_quotes_in_string_values() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string( - &mut classifier, - r#"{"name":"f","arguments":{"q":"he said \"hi\""}}"#, - 100, - ); - - assert!(outcomes.iter().all(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::ToolCall(LlamaToken::new(0))) - })); - } - - #[test] - fn json_probe_handles_unicode_letters_in_strings() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string( - &mut classifier, - r#"{"name":"日本語","arguments":{"city":"パリ"}}"#, - 100, - ); - - assert!(outcomes.iter().all(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::ToolCall(LlamaToken::new(0))) - })); - } - - #[test] - fn json_probe_handles_nested_objects() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string( - &mut classifier, - r#"{"name":"f","arguments":{"a":{"b":{"c":1}}}}"#, - 100, - ); - - assert!(outcomes.iter().all(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::ToolCall(LlamaToken::new(0))) - })); - } - - #[test] - fn json_probe_handles_arrays_inside_arguments() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string( - &mut classifier, - r#"{"name":"f","arguments":{"items":[1,2,3]}}"#, - 100, - ); - - assert!(outcomes.iter().all(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::ToolCall(LlamaToken::new(0))) - })); - } - - #[test] - fn json_probe_does_not_engage_when_first_byte_is_close_brace() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string(&mut classifier, "}}", 100); - - assert_eq!(classifier.probe_mode, ProbeMode::Idle); - assert!(outcomes.iter().all(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::Content(LlamaToken::new(0))) - })); - } - - #[test] - fn json_probe_does_not_engage_in_reasoning_section() { - let markers = StreamingMarkers::from_candidates([ - MarkerRoleCandidate { - tokens: vec![token(800)], - role: MarkerRole::ReasoningOpen, - }, - MarkerRoleCandidate { - tokens: vec![token(801)], - role: MarkerRole::ReasoningClose, - }, - MarkerRoleCandidate { - tokens: vec![token(900)], - role: MarkerRole::ToolCallOpen, - }, - ]) - .expect("synthetic markers must be valid"); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Reasoning; - - push_and_probe(&mut classifier, 1, "{"); - - assert_eq!(classifier.probe_mode, ProbeMode::Idle); - } - - #[test] - fn json_probe_does_not_engage_when_bare_json_tool_calls_are_ignored() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.bare_json_tool_calls = BareJsonToolCalls::Ignore; - classifier.section = SampledTokenSection::Content; - - let outcomes = push_and_probe(&mut classifier, 1, "{"); - - assert_eq!(classifier.probe_mode, ProbeMode::Idle); - assert_eq!(outcome_pieces(&outcomes), vec!["{"]); - } - - #[test] - fn json_probe_does_not_engage_in_tool_call_section() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::ToolCall; - - push_and_probe(&mut classifier, 1, "{"); - - assert_eq!(classifier.probe_mode, ProbeMode::Idle); - } - - #[test] - fn marker_probe_takes_precedence_when_both_could_match() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let mut outcomes = Vec::new(); - outcomes.extend(push_and_probe(&mut classifier, 1, "{")); - outcomes.extend(push_and_probe(&mut classifier, 900, r#"""#)); - - assert_eq!(classifier.section, SampledTokenSection::ToolCall); - assert_eq!(outcome_pieces(&outcomes), vec!["{", ""]); - assert_eq!( - outcome_sections(&outcomes), - vec![SampledTokenSection::Content, SampledTokenSection::ToolCall], - ); - } - - #[test] - fn json_probe_consumes_two_consecutive_objects_separately() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let mut outcomes = Vec::new(); - outcomes.extend(feed_json_string( - &mut classifier, - r#"{"name":"a","arguments":{}}"#, - 100, - )); - outcomes.extend(feed_json_string( - &mut classifier, - r#"{"name":"b","arguments":{"x":1}}"#, - 200, - )); - - let sections = outcome_sections(&outcomes); - assert!( - sections - .iter() - .all(|section| *section == SampledTokenSection::ToolCall), - "two consecutive markerless tool calls must both classify as ToolCall, got {sections:?}", - ); - } - - #[test] - fn json_probe_with_leading_whitespace_then_open_brace_classifies_whitespace_as_content_and_json_as_tool_call() - { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string( - &mut classifier, - "\n {\"name\":\"f\",\"arguments\":{}}", - 100, - ); - - let tool_call_count = outcomes - .iter() - .filter(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::ToolCall(LlamaToken::new(0))) - }) - .count(); - let content_count = outcomes - .iter() - .filter(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::Content(LlamaToken::new(0))) - }) - .count(); - assert_eq!( - content_count, 3, - "leading `\\n ` should classify as content" - ); - assert!( - tool_call_count > 0, - "the JSON object should classify as ToolCall", - ); - assert_eq!(content_count + tool_call_count, outcomes.len()); - } - - #[test] - fn json_probe_records_tool_call_token_usage_on_commit() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let json = r#"{"name":"f","arguments":{}}"#; - let outcomes = feed_json_string(&mut classifier, json, 100); - - let emitted = outcomes.len(); - let usage = classifier.usage(); - assert_eq!(usage.tool_call_tokens, emitted as u64); - assert_eq!(usage.content_tokens, 0); - } - - #[test] - fn json_probe_records_content_token_usage_on_abandon() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let json = r#"{"foo":"bar"}"#; - let outcomes = feed_json_string(&mut classifier, json, 100); - - let emitted = outcomes.len(); - let usage = classifier.usage(); - assert_eq!(usage.content_tokens, emitted as u64); - assert_eq!(usage.tool_call_tokens, 0); - } - - #[test] - fn finish_during_active_json_probe_releases_held_tokens_as_content() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - push_and_probe(&mut classifier, 1, "{"); - push_and_probe(&mut classifier, 2, r#""name""#); - assert_ne!(classifier.probe_mode, ProbeMode::Idle); - - let outcomes = finished(&mut classifier); - - let sections = outcome_sections(&outcomes); - assert!( - sections - .iter() - .all(|section| *section == SampledTokenSection::Content), - "finishing mid-probe must release held tokens as Content, got {sections:?}", - ); - assert_eq!(classifier.probe_mode, ProbeMode::Idle); - } - - #[test] - fn commit_probe_as_tool_call_requeues_non_held_entries_and_releases_held_as_tool_call() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - classifier.pending.push_back(PendingToken { - token: token(1), - decoded: "before".to_owned(), - section: SampledTokenSection::Content, - section_before_token: SampledTokenSection::Content, - marker_status: PendingMarkerStatus::Unmatched, - is_from_prompt: false, - }); - classifier.pending.push_back(PendingToken { - token: token(2), - decoded: "{}".to_owned(), - section: SampledTokenSection::Content, - section_before_token: SampledTokenSection::Content, - marker_status: PendingMarkerStatus::Unmatched, - is_from_prompt: false, - }); - classifier.probe_mode = ProbeMode::Active { - probe: StreamingJsonProbe::default(), - held_count: 1, - }; - - let mut outcomes = Vec::new(); - classifier.commit_probe_as_tool_call(&mut outcomes); - - let sections = outcome_sections(&outcomes); - assert_eq!(sections, vec![SampledTokenSection::ToolCall]); - assert_eq!(classifier.pending.len(), 1); - assert_eq!(classifier.pending[0].token, token(1)); - assert_eq!(classifier.probe_mode, ProbeMode::Idle); - } } diff --git a/llama-cpp-bindings/src/streaming_marker.rs b/llama-cpp-bindings/src/streaming_marker.rs deleted file mode 100644 index f2706d6be..000000000 --- a/llama-cpp-bindings/src/streaming_marker.rs +++ /dev/null @@ -1,91 +0,0 @@ -use crate::marker_role::MarkerRole; -use crate::sampled_token_section::SampledTokenSection; -use crate::token::LlamaToken; - -#[derive(Clone, Debug, Eq, PartialEq)] -/// A normalized token sequence and every semantic role attached to it. -pub struct StreamingMarker { - tokens: Vec, - roles: Vec, -} - -impl StreamingMarker { - #[must_use] - pub fn new(tokens: Vec, role: MarkerRole) -> Self { - Self { - tokens, - roles: vec![role], - } - } - - pub fn add_role(&mut self, role: MarkerRole) { - if !self.roles.contains(&role) { - self.roles.push(role); - } - } - - #[must_use] - /// Returns the tokens that form this marker. - pub fn tokens(&self) -> &[LlamaToken] { - &self.tokens - } - - #[must_use] - /// Returns the transitions associated with this marker. - pub fn roles(&self) -> &[MarkerRole] { - &self.roles - } - - #[must_use] - pub fn opener_count(&self) -> usize { - self.roles - .iter() - .filter(|role| role.opened_section().is_some()) - .count() - } - - fn opened_section(&self) -> Option { - self.roles.iter().find_map(|role| role.opened_section()) - } - - #[must_use] - pub fn span_section(&self, current: SampledTokenSection) -> SampledTokenSection { - self.opened_section().unwrap_or_else(|| { - if self - .roles - .iter() - .any(|role| role.closed_section() == Some(current)) - { - current - } else { - SampledTokenSection::Content - } - }) - } - - #[must_use] - pub fn next_section(&self) -> SampledTokenSection { - self.opened_section() - .unwrap_or(SampledTokenSection::Content) - } -} - -#[cfg(test)] -mod tests { - use super::StreamingMarker; - use crate::marker_role::MarkerRole; - use crate::token::LlamaToken; - - #[test] - fn a_role_the_marker_already_carries_is_not_added_twice() { - let mut marker = StreamingMarker::new(vec![LlamaToken::new(1)], MarkerRole::ReasoningClose); - - marker.add_role(MarkerRole::ToolCallOpen); - marker.add_role(MarkerRole::ToolCallOpen); - - assert_eq!( - marker.roles(), - &[MarkerRole::ReasoningClose, MarkerRole::ToolCallOpen] - ); - } -} diff --git a/llama-cpp-bindings/src/streaming_markers.rs b/llama-cpp-bindings/src/streaming_markers.rs deleted file mode 100644 index 8b9ed5e13..000000000 --- a/llama-cpp-bindings/src/streaming_markers.rs +++ /dev/null @@ -1,234 +0,0 @@ -use crate::error::MarkerDetectionError; -use crate::marker_role_candidate::MarkerRoleCandidate; -use crate::streaming_marker::StreamingMarker; -use crate::token::LlamaToken; - -#[derive(Clone, Debug, Default, Eq, PartialEq)] -/// The normalized streaming markers detected for a model. -pub struct StreamingMarkers { - markers: Vec, -} - -impl StreamingMarkers { - /// # Errors - /// - /// Returns [`MarkerDetectionError::EmptyMarker`] when a candidate carries no tokens, - /// and [`MarkerDetectionError::AmbiguousMarkerOpeners`] when one token sequence would - /// open more than one section. - pub fn from_candidates( - candidates: impl IntoIterator, - ) -> Result { - let mut markers: Vec = Vec::new(); - - for MarkerRoleCandidate { tokens, role } in candidates { - if tokens.is_empty() { - return Err(MarkerDetectionError::EmptyMarker); - } - - if let Some(marker) = markers.iter_mut().find(|marker| marker.tokens() == tokens) { - marker.add_role(role); - } else { - markers.push(StreamingMarker::new(tokens, role)); - } - } - - if let Some(marker) = markers.iter().find(|marker| marker.opener_count() > 1) { - return Err(MarkerDetectionError::AmbiguousMarkerOpeners { - tokens: marker.tokens().to_vec(), - }); - } - - Ok(Self { markers }) - } - - #[must_use] - /// Returns whether the model exposes no streaming markers. - pub const fn is_empty(&self) -> bool { - self.markers.is_empty() - } - - #[must_use] - /// Returns the number of distinct marker token sequences. - pub const fn len(&self) -> usize { - self.markers.len() - } - - /// Iterates over the distinct marker token sequences. - pub fn iter(&self) -> impl Iterator { - self.markers.iter() - } - - #[must_use] - pub fn max_token_len(&self) -> usize { - self.markers - .iter() - .map(|marker| marker.tokens().len()) - .max() - .unwrap_or(0) - } - - #[must_use] - pub fn longest_matching_suffix( - &self, - reversed_tokens: &TReversedTokens, - ) -> Option<&StreamingMarker> - where - TReversedTokens: Iterator + Clone, - { - self.markers - .iter() - .filter(|marker| { - marker - .tokens() - .iter() - .rev() - .copied() - .eq(reversed_tokens.clone().take(marker.tokens().len())) - }) - .max_by_key(|marker| marker.tokens().len()) - } - - #[must_use] - pub fn is_prefix_of_longer_marker(&self, tokens: &[LlamaToken]) -> bool { - self.markers.iter().any(|marker| { - marker.tokens().len() > tokens.len() && marker.tokens().starts_with(tokens) - }) - } -} - -#[cfg(test)] -mod tests { - use super::StreamingMarkers; - use crate::error::MarkerDetectionError; - use crate::marker_role::MarkerRole; - use crate::marker_role_candidate::MarkerRoleCandidate; - use crate::token::LlamaToken; - - fn token(id: i32) -> LlamaToken { - LlamaToken::new(id) - } - - #[test] - fn empty_collection_reports_no_markers() { - let markers = StreamingMarkers::default(); - - assert!(markers.is_empty()); - assert_eq!(markers.len(), 0); - assert_eq!(markers.max_token_len(), 0); - } - - #[test] - fn candidates_with_the_same_tokens_are_one_marker_with_multiple_roles() { - let markers = StreamingMarkers::from_candidates([ - MarkerRoleCandidate { - tokens: vec![token(1)], - role: MarkerRole::ReasoningClose, - }, - MarkerRoleCandidate { - tokens: vec![token(1)], - role: MarkerRole::ToolCallOpen, - }, - ]) - .expect("a close and an opener compose into one transition"); - - let marker = markers.iter().next().expect("one marker must remain"); - assert_eq!(marker.tokens(), &[token(1)]); - assert_eq!( - marker.roles(), - &[MarkerRole::ReasoningClose, MarkerRole::ToolCallOpen] - ); - } - - #[test] - fn empty_marker_is_rejected() { - assert_eq!( - StreamingMarkers::from_candidates([MarkerRoleCandidate { - tokens: Vec::new(), - role: MarkerRole::ReasoningOpen - }]), - Err(MarkerDetectionError::EmptyMarker) - ); - } - - #[test] - fn two_openers_for_the_same_tokens_are_rejected() { - let marker_tokens = vec![token(1), token(2)]; - - assert_eq!( - StreamingMarkers::from_candidates([ - MarkerRoleCandidate { - tokens: marker_tokens.clone(), - role: MarkerRole::ReasoningOpen - }, - MarkerRoleCandidate { - tokens: marker_tokens.clone(), - role: MarkerRole::ToolCallOpen - }, - ]), - Err(MarkerDetectionError::AmbiguousMarkerOpeners { - tokens: marker_tokens - }) - ); - } - - #[test] - fn longest_matching_suffix_wins() { - let markers = StreamingMarkers::from_candidates([ - MarkerRoleCandidate { - tokens: vec![token(2)], - role: MarkerRole::ReasoningClose, - }, - MarkerRoleCandidate { - tokens: vec![token(1), token(2)], - role: MarkerRole::ToolCallOpen, - }, - ]) - .expect("markers are valid"); - - let matched = markers - .longest_matching_suffix(&[token(1), token(2)].into_iter().rev()) - .expect("a suffix must match"); - - assert_eq!(matched.tokens(), &[token(1), token(2)]); - assert_eq!(matched.roles(), &[MarkerRole::ToolCallOpen]); - } - - #[test] - fn shorter_complete_marker_reports_when_it_is_still_an_ambiguous_prefix() { - let markers = StreamingMarkers::from_candidates([ - MarkerRoleCandidate { - tokens: vec![token(1)], - role: MarkerRole::ReasoningClose, - }, - MarkerRoleCandidate { - tokens: vec![token(1), token(2)], - role: MarkerRole::ToolCallOpen, - }, - ]) - .expect("markers are valid"); - - assert!(markers.is_prefix_of_longer_marker(&[token(1)])); - assert!(!markers.is_prefix_of_longer_marker(&[token(1), token(2)])); - } - - #[test] - fn max_token_len_uses_the_longest_normalized_marker() { - let markers = StreamingMarkers::from_candidates([ - MarkerRoleCandidate { - tokens: vec![token(1)], - role: MarkerRole::ReasoningOpen, - }, - MarkerRoleCandidate { - tokens: vec![token(2), token(3), token(4)], - role: MarkerRole::ReasoningClose, - }, - MarkerRoleCandidate { - tokens: vec![token(5), token(6)], - role: MarkerRole::ToolCallOpen, - }, - ]) - .expect("markers are valid"); - - assert_eq!(markers.max_token_len(), 3); - } -} diff --git a/llama-cpp-bindings/src/synthetic_tool_call_renders.rs b/llama-cpp-bindings/src/synthetic_tool_call_renders.rs deleted file mode 100644 index 2a14fa1da..000000000 --- a/llama-cpp-bindings/src/synthetic_tool_call_renders.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct SyntheticToolCallRenders { - pub without_tools: String, - pub with_tools: String, -} diff --git a/llama-cpp-bindings/src/tool_call_marker_pair.rs b/llama-cpp-bindings/src/tool_call_marker_pair.rs deleted file mode 100644 index 3ee5fd42b..000000000 --- a/llama-cpp-bindings/src/tool_call_marker_pair.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct ToolCallMarkerPair { - pub open: String, - pub close: String, -} diff --git a/llama-cpp-test-harness-macros/src/expand.rs b/llama-cpp-test-harness-macros/src/expand.rs index 787941c57..1e27074e0 100644 --- a/llama-cpp-test-harness-macros/src/expand.rs +++ b/llama-cpp-test-harness-macros/src/expand.rs @@ -1,10 +1,10 @@ -use proc_macro2::Ident; use proc_macro2::Span; use proc_macro2::TokenStream; use quote::quote; use syn::Item; use syn::ItemFn; use syn::ReturnType; +use syn::Signature; use syn::parse::Parse; use syn::parse::ParseStream; use syn::parse2; @@ -33,12 +33,6 @@ fn validate_signature(item_fn: &ItemFn) -> syn::Result<()> { "llama_test functions must take exactly one argument: `&LlamaFixture<'_>`", )); } - if matches!(item_fn.sig.output, ReturnType::Default) { - return Err(syn::Error::new_spanned( - &item_fn.sig, - "llama_test functions must return `anyhow::Result<()>`", - )); - } if item_fn.sig.asyncness.is_some() { return Err(syn::Error::new_spanned( &item_fn.sig, @@ -105,7 +99,21 @@ fn build_mmproj_source_literal(source: Option<&ParsedSource>) -> TokenStream { } } -fn build_registration(args: &ParsedArgs, fn_name: &Ident) -> TokenStream { +fn build_test_fn_literal(signature: &Signature) -> TokenStream { + let fn_name = &signature.ident; + + match signature.output { + ReturnType::Default => { + quote! { ::llama_cpp_test_harness::LlamaTestFn::Infallible(#fn_name) } + } + ReturnType::Type(..) => { + quote! { ::llama_cpp_test_harness::LlamaTestFn::Fallible(#fn_name) } + } + } +} + +fn build_registration(args: &ParsedArgs, signature: &Signature) -> TokenStream { + let fn_name = &signature.ident; let trial_name = format!( "{fn_name}[{suffix}]", suffix = args.model_source.display_suffix() @@ -120,6 +128,7 @@ fn build_registration(args: &ParsedArgs, fn_name: &Ident) -> TokenStream { let embeddings_flag = args.context_params.embeddings; let sequence_max = args.context_params.n_seq_max; let void_logs_flag = args.void_logs; + let test_fn = build_test_fn_literal(signature); let threads_batch = args.context_params.n_threads_batch.map_or_else( || quote! { ::core::option::Option::None }, |value| quote! { ::core::option::Option::Some(#value) }, @@ -146,7 +155,7 @@ fn build_registration(args: &ParsedArgs, fn_name: &Ident) -> TokenStream { embeddings: #embeddings_flag, }, void_logs: #void_logs_flag, - func: #fn_name, + func: #test_fn, } } } @@ -158,8 +167,7 @@ pub fn expand(attribute: TokenStream, item: TokenStream) -> syn::Result syn::Result TokenStream { quote! { @@ -192,245 +208,238 @@ mod tests { } } - #[test] - fn well_formed_input_expands_to_function_plus_submission() { - let expanded = expand(well_formed_attribute(), well_formed_function()) - .expect("well-formed input must expand") - .to_string(); + fn expansion_error(attribute: TokenStream, item: TokenStream) -> String { + expand(attribute, item) + .expect_err("the input must be refused") + .to_string() + } - assert!( - expanded.contains("fn my_test"), - "expansion missing the original fn: {expanded}" - ); - assert!( - expanded.contains("LlamaTestRegistration"), - "expansion missing LlamaTestRegistration: {expanded}", - ); - assert!( - expanded.contains("\"my_test[bar.gguf]\""), - "expansion missing the trial-name literal with file suffix: {expanded}", - ); - assert!( - expanded.contains("GgufSource :: HuggingFace"), - "expansion missing GgufSource::HuggingFace variant: {expanded}", - ); - assert!( - expanded.contains("func : my_test"), - "expansion missing func wire-up: {expanded}", - ); + fn test_fn_literal_of(item: TokenStream) -> String { + let item_fn: ItemFn = parse2(item).expect("the item must be a function"); + + build_test_fn_literal(&item_fn.sig).to_string() } #[test] - fn expansion_with_local_path_model_source_emits_local_variant() { - let attribute = quote! { - model_source = LocalPath("/abs/local.gguf"), - n_gpu_layers = 0, - load_mode = Mmap, - n_ctx = 1, - n_batch = 1, - n_ubatch = 1 + fn expansion_keeps_the_function_and_submits_its_registration() { + let expanded = expand(well_formed_attribute(), well_formed_function()) + .expect("well-formed input must expand"); + let expected = quote! { + fn my_test(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } + ::llama_cpp_test_harness::inventory::submit! { + ::llama_cpp_test_harness::LlamaTestRegistration { + name: "my_test[bar.gguf]", + key: ::llama_cpp_test_harness::LoadKey { + model_source: ::llama_cpp_test_harness::GgufSource::HuggingFace { + repo: "foo", + file: "bar.gguf", + }, + mmproj_source: ::core::option::Option::None, + model_load_params: ::llama_cpp_test_harness::ModelLoadParams { + n_gpu_layers: 0i32, + load_mode: ::llama_cpp_test_harness::LlamaLoadMode::Mmap, + }, + }, + context_params: ::llama_cpp_test_harness::ContextParams { + n_ctx: 1u32, + n_batch: 1u32, + n_ubatch: 1u32, + n_seq_max: 1u32, + n_threads_batch: ::core::option::Option::None, + embeddings: false, + }, + void_logs: false, + func: ::llama_cpp_test_harness::LlamaTestFn::Fallible(my_test), + } + } }; - let expanded = expand(attribute, well_formed_function()) - .expect("LocalPath must expand") - .to_string(); - assert!( - expanded.contains("GgufSource :: LocalPath"), - "expansion missing GgufSource::LocalPath variant: {expanded}", - ); - assert!( - expanded.contains("\"my_test[local.gguf]\""), - "trial name must use the path's filename component: {expanded}", - ); + assert_eq!(expanded.to_string(), expected.to_string()); } #[test] - fn expansion_with_mmproj_source_emits_some_variant() { - let attribute = quote! { - model_source = HuggingFace("r", "f"), - n_gpu_layers = 0, - load_mode = Mmap, - n_ctx = 1, - n_batch = 1, - n_ubatch = 1, - mmproj_source = HuggingFace("r", "mmproj.gguf") - }; - let expanded = expand(attribute, well_formed_function()) - .expect("mmproj_source must expand") - .to_string(); - - assert!( - expanded.contains("GgufSource :: HuggingFace"), - "expansion missing GgufSource::HuggingFace: {expanded}", - ); - assert!( - expanded.contains("Some"), - "expansion missing Option::Some wrap: {expanded}", + fn a_function_returning_a_result_registers_as_fallible() { + assert_eq!( + test_fn_literal_of(well_formed_function()), + quote! { ::llama_cpp_test_harness::LlamaTestFn::Fallible(my_test) }.to_string() ); } #[test] - fn expansion_with_local_path_mmproj_emits_local_variant() { - let attribute = quote! { - model_source = HuggingFace("r", "f"), - n_gpu_layers = 0, - load_mode = Mmap, - n_ctx = 1, - n_batch = 1, - n_ubatch = 1, - mmproj_source = LocalPath("/abs/mmproj.gguf") - }; - let expanded = expand(attribute, well_formed_function()) - .expect("LocalPath mmproj must expand") - .to_string(); - - assert!( - expanded.contains("GgufSource :: LocalPath"), - "expansion missing GgufSource::LocalPath: {expanded}", + fn a_function_without_a_return_type_registers_as_infallible() { + assert_eq!( + test_fn_literal_of(quote! { fn my_test(fixture: &LlamaFixture<'_>) {} }), + quote! { ::llama_cpp_test_harness::LlamaTestFn::Infallible(my_test) }.to_string() ); } #[test] - fn expansion_without_mmproj_emits_none_variant() { - let expanded = expand(well_formed_attribute(), well_formed_function()) - .expect("no-mmproj must expand") - .to_string(); - - assert!( - expanded.contains("None"), - "expansion missing Option::None for absent mmproj: {expanded}" + fn a_local_path_model_source_emits_the_local_path_variant() { + assert_eq!( + build_model_source_literal(&ParsedSource::LocalPath("/abs/local.gguf".to_owned())) + .to_string(), + quote! { ::llama_cpp_test_harness::GgufSource::LocalPath("/abs/local.gguf") } + .to_string() ); } #[test] - fn malformed_attribute_propagates_parser_error() { - let attribute = quote! { totally_wrong = "x" }; - let error = expand(attribute, well_formed_function()) - .expect_err("malformed must fail") - .to_string(); - - assert!(error.contains("unknown field"), "got: {error}"); + fn a_huggingface_mmproj_source_is_wrapped_in_some() { + assert_eq!( + build_mmproj_source_literal(Some(&ParsedSource::HuggingFace { + repo: "r".to_owned(), + file: "mmproj.gguf".to_owned(), + })) + .to_string(), + quote! { + ::core::option::Option::Some(::llama_cpp_test_harness::GgufSource::HuggingFace { + repo: "r", + file: "mmproj.gguf", + }) + } + .to_string() + ); } #[test] - fn function_with_too_many_arguments_is_rejected() { - let item = quote! { - fn my_test(a: &LlamaFixture<'_>, b: i32) -> anyhow::Result<()> { Ok(()) } - }; - let error = expand(well_formed_attribute(), item) - .expect_err("two-arg must fail") - .to_string(); - - assert!(error.contains("exactly one argument"), "got: {error}"); + fn a_local_path_mmproj_source_is_wrapped_in_some() { + assert_eq!( + build_mmproj_source_literal(Some(&ParsedSource::LocalPath( + "/abs/mmproj.gguf".to_owned() + ))) + .to_string(), + quote! { + ::core::option::Option::Some(::llama_cpp_test_harness::GgufSource::LocalPath( + "/abs/mmproj.gguf" + )) + } + .to_string() + ); } #[test] - fn function_with_zero_arguments_is_rejected() { - let item = quote! { fn my_test() -> anyhow::Result<()> { Ok(()) } }; - let error = expand(well_formed_attribute(), item) - .expect_err("zero-arg must fail") - .to_string(); - - assert!(error.contains("exactly one argument"), "got: {error}"); + fn a_malformed_attribute_propagates_the_parser_error() { + assert_eq!( + expansion_error(quote! { totally_wrong = "x" }, well_formed_function()), + "unknown field `totally_wrong`; expected one of: model_source, n_gpu_layers, load_mode, \ + n_ctx, n_batch, n_ubatch, mmproj_source, embeddings, n_seq_max, n_threads_batch, \ + void_logs" + ); } #[test] - fn impl_block_input_is_rejected_at_parse_time() { - let item = quote! { - impl S { - fn my_test(&self) -> anyhow::Result<()> { Ok(()) } - } - }; - let result = expand(well_formed_attribute(), item); - - assert!(result.is_err(), "impl block must fail"); + fn a_function_with_two_arguments_is_refused() { + assert_eq!( + expansion_error( + well_formed_attribute(), + quote! { fn my_test(a: &LlamaFixture<'_>, b: i32) -> anyhow::Result<()> { Ok(()) } }, + ), + "llama_test functions must take exactly one argument: `&LlamaFixture<'_>`" + ); } #[test] - fn function_without_return_type_is_rejected() { - let item = quote! { fn my_test(fixture: &LlamaFixture<'_>) { } }; - let error = expand(well_formed_attribute(), item) - .expect_err("missing return type must fail") - .to_string(); - - assert!(error.contains("anyhow::Result"), "got: {error}"); + fn a_function_without_arguments_is_refused() { + assert_eq!( + expansion_error( + well_formed_attribute(), + quote! { fn my_test() -> anyhow::Result<()> { Ok(()) } }, + ), + "llama_test functions must take exactly one argument: `&LlamaFixture<'_>`" + ); } #[test] - fn async_function_is_rejected() { - let item = quote! { - async fn my_test(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } - }; - let error = expand(well_formed_attribute(), item) - .expect_err("async must fail") - .to_string(); - - assert!(error.contains("synchronous"), "got: {error}"); + fn an_async_function_is_refused() { + assert_eq!( + expansion_error( + well_formed_attribute(), + quote! { + async fn my_test(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } + }, + ), + "llama_test functions must be synchronous" + ); } #[test] - fn generic_function_is_rejected() { - let item = quote! { - fn my_test(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } - }; - let error = expand(well_formed_attribute(), item) - .expect_err("generic must fail") - .to_string(); - - assert!(error.contains("generic"), "got: {error}"); + fn a_generic_function_is_refused() { + assert_eq!( + expansion_error( + well_formed_attribute(), + quote! { + fn my_test(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } + }, + ), + "llama_test functions must not be generic" + ); } #[test] - fn malformed_item_token_stream_is_rejected() { - let item = quote! { this is not a function }; - let result = expand(well_formed_attribute(), item); - - assert!(result.is_err(), "non-fn item must fail"); + fn an_impl_block_is_refused() { + assert_eq!( + expansion_error( + well_formed_attribute(), + quote! { + impl S { + fn my_test(&self) -> anyhow::Result<()> { Ok(()) } + } + }, + ), + "llama_test expects an fn definition" + ); } #[test] - fn stacked_invocation_preserves_prior_submission() { - let prior_layer_output = expand(well_formed_attribute(), well_formed_function()) - .expect("first layer must expand"); - - let second_attribute = quote! { - model_source = HuggingFace("second", "second.gguf"), - n_gpu_layers = 1, - load_mode = None, - n_ctx = 2, - n_batch = 2, - n_ubatch = 2 - }; - - let second_layer_output = expand(second_attribute, prior_layer_output) - .expect("stacked second invocation must expand") - .to_string(); - - assert!( - second_layer_output.contains("\"my_test[bar.gguf]\""), - "stacked output missing first trial name: {second_layer_output}", - ); - assert!( - second_layer_output.contains("\"my_test[second.gguf]\""), - "stacked output missing second trial name: {second_layer_output}", - ); - let occurrences = second_layer_output.matches("LlamaTestRegistration").count(); - assert!( - occurrences >= 2, - "stacked output should contain two LlamaTestRegistration submissions, found {occurrences}: {second_layer_output}", + fn tokens_that_are_not_an_item_are_refused() { + assert_eq!( + expansion_error(well_formed_attribute(), quote! { this is not a function }), + "expected `!`" ); } #[test] - fn two_fn_definitions_in_input_are_rejected() { - let item = quote! { - fn first(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } - fn second(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } - }; - let error = expand(well_formed_attribute(), item) - .expect_err("two fns must fail") - .to_string(); + fn two_function_definitions_are_refused() { + assert_eq!( + expansion_error( + well_formed_attribute(), + quote! { + fn first(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } + fn second(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } + }, + ), + "llama_test expects exactly one fn definition" + ); + } - assert!(error.contains("exactly one fn"), "got: {error}"); + #[test] + fn a_stacked_invocation_keeps_the_registration_of_the_previous_layer() { + let first_layer = expand(well_formed_attribute(), well_formed_function()) + .expect("the first layer must expand"); + let second_layer = expand( + quote! { + model_source = HuggingFace("second", "second.gguf"), + n_gpu_layers = 1, + load_mode = None, + n_ctx = 2, + n_batch = 2, + n_ubatch = 2 + }, + first_layer, + ) + .expect("the stacked layer must expand"); + let StackedItems { items } = parse2(second_layer).expect("the expansion must be items"); + + assert_eq!( + items + .iter() + .map(|item| match item { + Item::Fn(_) => "fn", + Item::Macro(_) => "registration", + _ => "other", + }) + .collect::>(), + ["fn", "registration", "registration"] + ); } } diff --git a/llama-cpp-test-harness/src/execution_plan.rs b/llama-cpp-test-harness/src/execution_plan.rs index e1c203a54..75d0d32e8 100644 --- a/llama-cpp-test-harness/src/execution_plan.rs +++ b/llama-cpp-test-harness/src/execution_plan.rs @@ -76,6 +76,7 @@ mod tests { use crate::LlamaLoadMode; use crate::context_params::ContextParams; use crate::gguf_source::GgufSource; + use crate::llama_test_fn::LlamaTestFn; use crate::llama_test_registration::LlamaTestRegistration; use crate::load_key::LoadKey; use crate::model_load_params::ModelLoadParams; @@ -116,7 +117,7 @@ mod tests { }, context_params: TRIVIAL_CONTEXT_PARAMS, void_logs: false, - func: no_op, + func: LlamaTestFn::Infallible(no_op), }; static REG_BETA_B: LlamaTestRegistration = LlamaTestRegistration { name: "bravo", @@ -133,7 +134,7 @@ mod tests { }, context_params: TRIVIAL_CONTEXT_PARAMS, void_logs: false, - func: no_op, + func: LlamaTestFn::Infallible(no_op), }; static REG_ALPHA_Z: LlamaTestRegistration = LlamaTestRegistration { name: "zulu", @@ -150,7 +151,7 @@ mod tests { }, context_params: TRIVIAL_CONTEXT_PARAMS, void_logs: false, - func: no_op, + func: LlamaTestFn::Infallible(no_op), }; static REG_BETA_DIFFERENT_CONTEXT: LlamaTestRegistration = LlamaTestRegistration { name: "charlie", @@ -167,7 +168,7 @@ mod tests { }, context_params: ALTERNATE_CONTEXT_PARAMS, void_logs: false, - func: no_op, + func: LlamaTestFn::Infallible(no_op), }; static REG_VOID_LOGS: LlamaTestRegistration = LlamaTestRegistration { @@ -185,7 +186,7 @@ mod tests { }, context_params: TRIVIAL_CONTEXT_PARAMS, void_logs: true, - func: no_op, + func: LlamaTestFn::Infallible(no_op), }; #[test] diff --git a/llama-cpp-test-harness/src/llama_test_fn.rs b/llama-cpp-test-harness/src/llama_test_fn.rs index f49249b6e..38521d21b 100644 --- a/llama-cpp-test-harness/src/llama_test_fn.rs +++ b/llama-cpp-test-harness/src/llama_test_fn.rs @@ -1,4 +1,28 @@ use crate::llama_fixture::LlamaFixture; -pub type LlamaTestFn = - for<'reference, 'fixture> fn(&'reference LlamaFixture<'fixture>) -> anyhow::Result<()>; +#[derive(Clone, Copy)] +pub enum LlamaTestFn { + Fallible( + for<'reference, 'fixture> fn(&'reference LlamaFixture<'fixture>) -> anyhow::Result<()>, + ), + Infallible(for<'reference, 'fixture> fn(&'reference LlamaFixture<'fixture>)), +} + +impl LlamaTestFn { + /// Runs the test against `fixture`; a test that cannot fail by returning an error succeeds + /// whenever it returns. + /// + /// # Errors + /// + /// Forwards the error a fallible test returns. + pub fn run(self, fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { + match self { + Self::Fallible(test) => test(fixture), + Self::Infallible(test) => { + test(fixture); + + Ok(()) + } + } + } +} diff --git a/llama-cpp-test-harness/src/no_op.rs b/llama-cpp-test-harness/src/no_op.rs index 95c62c541..1a746c447 100644 --- a/llama-cpp-test-harness/src/no_op.rs +++ b/llama-cpp-test-harness/src/no_op.rs @@ -1,8 +1,3 @@ use crate::llama_fixture::LlamaFixture; -/// # Errors -/// -/// Never; always returns `Ok(())`. The `Result` return type matches `LlamaTestFn`. -pub const fn no_op(_fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { - Ok(()) -} +pub const fn no_op(_fixture: &LlamaFixture<'_>) {} diff --git a/llama-cpp-test-harness/src/scheduled_phase.rs b/llama-cpp-test-harness/src/scheduled_phase.rs index 5c4dfeef1..c743fcce4 100644 --- a/llama-cpp-test-harness/src/scheduled_phase.rs +++ b/llama-cpp-test-harness/src/scheduled_phase.rs @@ -15,14 +15,16 @@ fn trial(registration: &'static LlamaTestRegistration, setup: &Arc) let setup = Arc::clone(setup); Trial::test(registration.name, move || match setup.load_phase_state() { - Ok(state) => (registration.func)(&LlamaFixture { - model: &state.model, - backend: &state.backend, - context_params: ®istration.context_params, - mtmd_context: state.mtmd_context.as_ref(), - model_path: &state.model_path, - }) - .map_err(|error| Failed::from(format!("{error:#}"))), + Ok(state) => registration + .func + .run(&LlamaFixture { + model: &state.model, + backend: &state.backend, + context_params: ®istration.context_params, + mtmd_context: state.mtmd_context.as_ref(), + model_path: &state.model_path, + }) + .map_err(|error| Failed::from(format!("{error:#}"))), Err(setup_failure) => Err(Failed::from(format!( "phase setup failed: {setup_failure:#}" ))), @@ -95,6 +97,7 @@ mod tests { use crate::context_params::ContextParams; use crate::execution_phase::ExecutionPhase; use crate::gguf_source::GgufSource; + use crate::llama_test_fn::LlamaTestFn; use crate::llama_test_registration::LlamaTestRegistration; use crate::load_key::LoadKey; use crate::model_load_params::ModelLoadParams; @@ -122,7 +125,7 @@ mod tests { embeddings: false, }, void_logs: false, - func: no_op, + func: LlamaTestFn::Infallible(no_op), }; fn selects_any_trial_with(harness_arguments: &[&str]) -> bool { diff --git a/llama-cpp-test-harness/tests/harness_self_test.rs b/llama-cpp-test-harness/tests/harness_self_test.rs index 42e4f7b8c..ba026416e 100644 --- a/llama-cpp-test-harness/tests/harness_self_test.rs +++ b/llama-cpp-test-harness/tests/harness_self_test.rs @@ -1,7 +1,3 @@ -#![expect( - clippy::unnecessary_wraps, - reason = "harness trial functions must return Result to satisfy LlamaTestFn registration" -)] use std::process::ExitCode; use anyhow::Result; @@ -20,10 +16,10 @@ use llama_cpp_test_harness::run_to_conclusions; n_ubatch = 64, void_logs = true, )] -fn phase_a_first_passing_trial(fixture: &LlamaFixture<'_>) -> Result<()> { +fn phase_a_first_passing_trial(fixture: &LlamaFixture<'_>) { let formatted = format!("{:?}", fixture.model); assert!(formatted.contains("LlamaModel")); - no_op(fixture) + no_op(fixture); } #[llama_test( @@ -34,9 +30,8 @@ fn phase_a_first_passing_trial(fixture: &LlamaFixture<'_>) -> Result<()> { n_batch = 128, n_ubatch = 64 )] -fn phase_a_second_passing_trial(fixture: &LlamaFixture<'_>) -> Result<()> { +fn phase_a_second_passing_trial(fixture: &LlamaFixture<'_>) { assert_eq!(fixture.context_params.n_ctx, 512); - Ok(()) } #[llama_test( @@ -59,9 +54,8 @@ fn phase_a_intentionally_failing_trial(_fixture: &LlamaFixture<'_>) -> Result<() n_batch = 128, n_ubatch = 64 )] -fn phase_b_first_passing_trial(fixture: &LlamaFixture<'_>) -> Result<()> { +fn phase_b_first_passing_trial(fixture: &LlamaFixture<'_>) { assert!(fixture.mtmd_context.is_none()); - Ok(()) } #[llama_test( @@ -72,9 +66,8 @@ fn phase_b_first_passing_trial(fixture: &LlamaFixture<'_>) -> Result<()> { n_batch = 128, n_ubatch = 64 )] -fn phase_b_second_passing_trial(fixture: &LlamaFixture<'_>) -> Result<()> { - let _markers = fixture.model.tool_call_markers(); - Ok(()) +fn phase_b_second_passing_trial(fixture: &LlamaFixture<'_>) { + assert_eq!(fixture.context_params.n_ubatch, 64); } #[llama_test( @@ -119,9 +112,8 @@ fn phase_b_second_passing_trial(fixture: &LlamaFixture<'_>) -> Result<()> { n_batch = 1, n_ubatch = 1 )] -fn shared_setup_failure_and_phase_a_trial(fixture: &LlamaFixture<'_>) -> Result<()> { +fn shared_setup_failure_and_phase_a_trial(fixture: &LlamaFixture<'_>) { assert!(fixture.model_path.exists()); - Ok(()) } #[llama_test( @@ -133,12 +125,11 @@ fn shared_setup_failure_and_phase_a_trial(fixture: &LlamaFixture<'_>) -> Result< n_ubatch = 64, mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf") )] -fn phase_d_mmproj_trial(fixture: &LlamaFixture<'_>) -> Result<()> { +fn phase_d_mmproj_trial(fixture: &LlamaFixture<'_>) { assert!( fixture.mtmd_context.is_some(), "mmproj_file declared, but fixture.mtmd_context is None", ); - Ok(()) } const EXPECTED_PHASES: usize = 7; diff --git a/llama-cpp-wrapper-sources/src/wrapper_headers.rs b/llama-cpp-wrapper-sources/src/wrapper_headers.rs index a366ebd1f..958275a2a 100644 --- a/llama-cpp-wrapper-sources/src/wrapper_headers.rs +++ b/llama-cpp-wrapper-sources/src/wrapper_headers.rs @@ -1,17 +1,18 @@ pub const WRAPPER_HEADERS: &[&str] = &[ "wrapper.h", - "wrapper_chat_apply.h", - "wrapper_chat_parse.h", + "wrapper_batch.h", + "wrapper_chat_session.h", + "wrapper_chat_templates.h", + "wrapper_chat_templates_handle.h", "wrapper_common.h", "wrapper_context.h", "wrapper_fit.h", "wrapper_gbnf.h", "wrapper_mtmd.h", - "wrapper_reasoning.h", + "wrapper_parsed_chat.h", + "wrapper_parsed_chat_handle.h", "wrapper_report_current_exception.h", "wrapper_sampler_init.h", "wrapper_state.h", - "wrapper_token_text.h", - "wrapper_tool_calls.h", "wrapper_utils.h", ]; diff --git a/llama-cpp-wrapper-sources/src/wrapper_sources.rs b/llama-cpp-wrapper-sources/src/wrapper_sources.rs index b9bd9ab98..06ac02144 100644 --- a/llama-cpp-wrapper-sources/src/wrapper_sources.rs +++ b/llama-cpp-wrapper-sources/src/wrapper_sources.rs @@ -1,15 +1,13 @@ pub const WRAPPER_SOURCES: &[&str] = &[ "wrapper_batch.cpp", - "wrapper_chat_apply.cpp", - "wrapper_chat_parse.cpp", + "wrapper_chat_session.cpp", + "wrapper_chat_templates.cpp", "wrapper_common.cpp", "wrapper_context.cpp", "wrapper_fit.cpp", "wrapper_gbnf.cpp", "wrapper_mtmd.cpp", - "wrapper_reasoning.cpp", + "wrapper_parsed_chat.cpp", "wrapper_sampler_init.cpp", "wrapper_state.cpp", - "wrapper_token_text.cpp", - "wrapper_tool_calls.cpp", ]; diff --git a/llama-cpp-wrapper-sources/tests/wrapper_files_on_disk.rs b/llama-cpp-wrapper-sources/tests/wrapper_files_on_disk.rs new file mode 100644 index 000000000..3890ce356 --- /dev/null +++ b/llama-cpp-wrapper-sources/tests/wrapper_files_on_disk.rs @@ -0,0 +1,39 @@ +use std::collections::BTreeSet; +use std::ffi::OsStr; +use std::fs; +use std::path::Path; + +use llama_cpp_wrapper_sources::wrapper_headers::WRAPPER_HEADERS; +use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; + +const SYS_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../llama-cpp-bindings-sys"); + +fn sys_root_files_with_extension(extension: &str) -> BTreeSet { + fs::read_dir(Path::new(SYS_DIR)) + .expect("the sys crate directory must be readable") + .map(|entry| { + entry + .expect("a sys crate directory entry must be readable") + .path() + }) + .filter(|path| path.is_file() && path.extension() == Some(OsStr::new(extension))) + .filter_map(|path| path.file_name()?.to_str().map(str::to_owned)) + .collect() +} + +fn listed(files: &[&str]) -> BTreeSet { + files.iter().map(|file| (*file).to_owned()).collect() +} + +#[test] +fn every_cpp_source_in_the_sys_crate_root_is_a_listed_wrapper_source() { + assert_eq!( + sys_root_files_with_extension("cpp"), + listed(WRAPPER_SOURCES) + ); +} + +#[test] +fn every_header_in_the_sys_crate_root_is_a_listed_wrapper_header() { + assert_eq!(sys_root_files_with_extension("h"), listed(WRAPPER_HEADERS)); +} From fd79de36e46e7bfaed356b8dd849b5ec5ea03132 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Fri, 30 Oct 2026 12:08:43 +0100 Subject: [PATCH 03/82] Disable the clang-tidy analyzer check that misreports valid bitmask enum values --- .clang-tidy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.clang-tidy b/.clang-tidy index 73eb4e2d7..cf54a5bba 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -12,6 +12,7 @@ Checks: > -mpi-*, -openmp-*, -zircon-*, - -bugprone-easily-swappable-parameters + -bugprone-easily-swappable-parameters, + -clang-analyzer-optin.core.EnumCastOutOfRange WarningsAsErrors: '*' FormatStyle: none From 95e60ad23f9227c391e5450fb818d73aa8bb2ead Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Fri, 30 Oct 2026 12:53:23 +0100 Subject: [PATCH 04/82] Cover chat session, FFI status mappings and recurrent memory edits with tests and drop unreachable destructor statuses --- .../wrapper_chat_session.cpp | 49 +- llama-cpp-bindings-sys/wrapper_chat_session.h | 29 +- .../wrapper_chat_templates.cpp | 19 +- .../wrapper_chat_templates.h | 11 +- .../wrapper_parsed_chat.cpp | 19 +- llama-cpp-bindings-sys/wrapper_parsed_chat.h | 11 +- .../tests/chat_prompt_rendering.rs | 57 ++ .../tests/kv_cache_sequence_edits.rs | 28 + llama-cpp-bindings-tests/tests/main.rs | 1 + .../tests/multimodal_chunk_evaluation.rs | 54 ++ .../tests/prompt_token_accounting.rs | 23 + .../tests/sampled_token_ingestion.rs | 40 ++ .../tests/tool_call_grammar.rs | 136 +++++ .../src/bare_json_tool_call_probe.rs | 59 +- llama-cpp-bindings/src/chat_classification.rs | 49 +- .../src/chat_grammar_trigger.rs | 77 +++ .../src/chat_message_parse_outcome.rs | 70 +++ llama-cpp-bindings/src/chat_session.rs | 554 ++++-------------- llama-cpp-bindings/src/chat_templates.rs | 90 +-- llama-cpp-bindings/src/context.rs | 324 ++++++++++ llama-cpp-bindings/src/context/kv_cache.rs | 87 +++ llama-cpp-bindings/src/context/session.rs | 189 +++++- .../src/error/parse_chat_message_error.rs | 2 - llama-cpp-bindings/src/grammar_matcher.rs | 32 +- .../src/json_schema_to_grammar.rs | 21 + .../src/lazy_grammar_triggers.rs | 258 ++++++++ llama-cpp-bindings/src/lib.rs | 2 +- llama-cpp-bindings/src/llguidance_sampler.rs | 125 +++- llama-cpp-bindings/src/model.rs | 44 +- llama-cpp-bindings/src/model/params.rs | 43 +- .../src/mtmd/fitted_mtmd_input_chunk.rs | 8 - llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs | 38 ++ llama-cpp-bindings/src/mtmd/mtmd_context.rs | 53 ++ llama-cpp-bindings/src/parsed_chat.rs | 126 +--- .../src/result_after_release.rs | 50 -- llama-cpp-bindings/src/sampled_token.rs | 46 ++ .../src/sampled_token_classifier.rs | 35 +- llama-cpp-bindings/src/sampling.rs | 151 ++++- .../src/streaming_json_probe.rs | 26 + llama-cpp-bindings/src/token/data_array.rs | 16 + .../src/tool_call_format/paired_quote_args.rs | 19 + llama-cpp-test-harness-macros/src/expand.rs | 17 +- 42 files changed, 2130 insertions(+), 958 deletions(-) create mode 100644 llama-cpp-bindings-tests/tests/tool_call_grammar.rs create mode 100644 llama-cpp-bindings/src/lazy_grammar_triggers.rs delete mode 100644 llama-cpp-bindings/src/result_after_release.rs diff --git a/llama-cpp-bindings-sys/wrapper_chat_session.cpp b/llama-cpp-bindings-sys/wrapper_chat_session.cpp index 2430db17a..728b6a41b 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_session.cpp +++ b/llama-cpp-bindings-sys/wrapper_chat_session.cpp @@ -241,23 +241,8 @@ extern "C" auto llama_rs_chat_session_create( } } -extern "C" auto llama_rs_chat_session_free( - llama_rs_chat_session_handle session, - char ** out_error) -> llama_rs_chat_session_free_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - try { - const std::unique_ptr reclaimed(session); - - return LLAMA_RS_CHAT_SESSION_FREE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_CHAT_SESSION_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_SESSION_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - LLAMA_RS_CHAT_SESSION_FREE_ERROR_STRING_ALLOCATION_FAILED); - } +extern "C" auto llama_rs_chat_session_free(llama_rs_chat_session_handle session) -> void { + const std::unique_ptr reclaimed(session); } extern "C" auto llama_rs_chat_session_prompt_part_count(llama_rs_chat_session_handle session) -> size_t { @@ -356,17 +341,6 @@ extern "C" auto llama_rs_chat_session_thinking_end_tag( return llama_rs_string_view(session->session.thinking_end_tags().at(index), out_length); } -extern "C" auto llama_rs_chat_session_additional_stop_count(llama_rs_chat_session_handle session) -> size_t { - return session->session.additional_stops().size(); -} - -extern "C" auto llama_rs_chat_session_additional_stop( - llama_rs_chat_session_handle session, - size_t index, - size_t * out_length) -> const char * { - return llama_rs_string_view(session->session.additional_stops().at(index), out_length); -} - extern "C" auto llama_rs_chat_session_parse( llama_rs_chat_session_handle session, const char * text, @@ -523,23 +497,8 @@ extern "C" auto llama_rs_chat_spans_end(llama_rs_chat_spans_handle spans, size_t return spans->spans.at(index).end; } -extern "C" auto llama_rs_chat_spans_free( - llama_rs_chat_spans_handle spans, - char ** out_error) -> llama_rs_chat_spans_free_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - try { - const std::unique_ptr reclaimed(spans); - - return LLAMA_RS_CHAT_SPANS_FREE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_CHAT_SPANS_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_SPANS_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - LLAMA_RS_CHAT_SPANS_FREE_ERROR_STRING_ALLOCATION_FAILED); - } +extern "C" auto llama_rs_chat_spans_free(llama_rs_chat_spans_handle spans) -> void { + const std::unique_ptr reclaimed(spans); } extern "C" auto llama_rs_regex_escape( diff --git a/llama-cpp-bindings-sys/wrapper_chat_session.h b/llama-cpp-bindings-sys/wrapper_chat_session.h index ee8f2c8e0..1bfd36354 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_session.h +++ b/llama-cpp-bindings-sys/wrapper_chat_session.h @@ -65,16 +65,7 @@ llama_rs_chat_session_create_status llama_rs_chat_session_create( llama_rs_chat_session_handle * out_session, char ** out_error); -typedef enum llama_rs_chat_session_free_status { - LLAMA_RS_CHAT_SESSION_FREE_OK = 0, - LLAMA_RS_CHAT_SESSION_FREE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_CHAT_SESSION_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_SESSION_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, -} llama_rs_chat_session_free_status; - -llama_rs_chat_session_free_status llama_rs_chat_session_free( - llama_rs_chat_session_handle session, - char ** out_error); +void llama_rs_chat_session_free(llama_rs_chat_session_handle session); size_t llama_rs_chat_session_prompt_part_count(llama_rs_chat_session_handle session); @@ -128,13 +119,6 @@ const char * llama_rs_chat_session_thinking_end_tag( size_t index, size_t * out_length); -size_t llama_rs_chat_session_additional_stop_count(llama_rs_chat_session_handle session); - -const char * llama_rs_chat_session_additional_stop( - llama_rs_chat_session_handle session, - size_t index, - size_t * out_length); - typedef enum llama_rs_chat_session_parse_status { LLAMA_RS_CHAT_SESSION_PARSE_OK = 0, LLAMA_RS_CHAT_SESSION_PARSE_NULL_SESSION_ARG, @@ -197,16 +181,7 @@ size_t llama_rs_chat_spans_start(llama_rs_chat_spans_handle spans, size_t index) size_t llama_rs_chat_spans_end(llama_rs_chat_spans_handle spans, size_t index); -typedef enum llama_rs_chat_spans_free_status { - LLAMA_RS_CHAT_SPANS_FREE_OK = 0, - LLAMA_RS_CHAT_SPANS_FREE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_CHAT_SPANS_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_SPANS_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, -} llama_rs_chat_spans_free_status; - -llama_rs_chat_spans_free_status llama_rs_chat_spans_free( - llama_rs_chat_spans_handle spans, - char ** out_error); +void llama_rs_chat_spans_free(llama_rs_chat_spans_handle spans); typedef enum llama_rs_regex_escape_status { LLAMA_RS_REGEX_ESCAPE_OK = 0, diff --git a/llama-cpp-bindings-sys/wrapper_chat_templates.cpp b/llama-cpp-bindings-sys/wrapper_chat_templates.cpp index e86d80f64..45d3a6dc3 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_templates.cpp +++ b/llama-cpp-bindings-sys/wrapper_chat_templates.cpp @@ -64,23 +64,8 @@ extern "C" auto llama_rs_chat_templates_init( } } -extern "C" auto llama_rs_chat_templates_free( - llama_rs_chat_templates_handle templates, - char ** out_error) -> llama_rs_chat_templates_free_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - try { - const std::unique_ptr reclaimed(templates); - - return LLAMA_RS_CHAT_TEMPLATES_FREE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_CHAT_TEMPLATES_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_TEMPLATES_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - LLAMA_RS_CHAT_TEMPLATES_FREE_ERROR_STRING_ALLOCATION_FAILED); - } +extern "C" auto llama_rs_chat_templates_free(llama_rs_chat_templates_handle templates) -> void { + const std::unique_ptr reclaimed(templates); } extern "C" auto llama_rs_chat_templates_supports_tool_calls(llama_rs_chat_templates_handle templates) -> bool { diff --git a/llama-cpp-bindings-sys/wrapper_chat_templates.h b/llama-cpp-bindings-sys/wrapper_chat_templates.h index c7a1b8f2e..9533a0062 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_templates.h +++ b/llama-cpp-bindings-sys/wrapper_chat_templates.h @@ -27,16 +27,7 @@ llama_rs_chat_templates_init_status llama_rs_chat_templates_init( llama_rs_chat_templates_handle * out_templates, char ** out_error); -typedef enum llama_rs_chat_templates_free_status { - LLAMA_RS_CHAT_TEMPLATES_FREE_OK = 0, - LLAMA_RS_CHAT_TEMPLATES_FREE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_CHAT_TEMPLATES_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_TEMPLATES_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, -} llama_rs_chat_templates_free_status; - -llama_rs_chat_templates_free_status llama_rs_chat_templates_free( - llama_rs_chat_templates_handle templates, - char ** out_error); +void llama_rs_chat_templates_free(llama_rs_chat_templates_handle templates); bool llama_rs_chat_templates_supports_tool_calls(llama_rs_chat_templates_handle templates); diff --git a/llama-cpp-bindings-sys/wrapper_parsed_chat.cpp b/llama-cpp-bindings-sys/wrapper_parsed_chat.cpp index 43a7b0dfc..f9f32c99a 100644 --- a/llama-cpp-bindings-sys/wrapper_parsed_chat.cpp +++ b/llama-cpp-bindings-sys/wrapper_parsed_chat.cpp @@ -1,27 +1,12 @@ #include "wrapper_parsed_chat.h" #include "wrapper_parsed_chat_handle.h" -#include "wrapper_report_current_exception.h" #include "wrapper_utils.h" #include #include -extern "C" auto llama_rs_parsed_chat_free( - llama_rs_parsed_chat_handle handle, - char ** out_error) -> llama_rs_parsed_chat_free_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - try { - const std::unique_ptr reclaimed(handle); - return LLAMA_RS_PARSED_CHAT_FREE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_PARSED_CHAT_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - LLAMA_RS_PARSED_CHAT_FREE_ERROR_STRING_ALLOCATION_FAILED); - } +extern "C" auto llama_rs_parsed_chat_free(llama_rs_parsed_chat_handle handle) -> void { + const std::unique_ptr reclaimed(handle); } extern "C" auto llama_rs_parsed_chat_content(llama_rs_parsed_chat_handle handle, size_t * out_length) diff --git a/llama-cpp-bindings-sys/wrapper_parsed_chat.h b/llama-cpp-bindings-sys/wrapper_parsed_chat.h index 1cc3e5a24..6cd20e6c0 100644 --- a/llama-cpp-bindings-sys/wrapper_parsed_chat.h +++ b/llama-cpp-bindings-sys/wrapper_parsed_chat.h @@ -11,16 +11,7 @@ extern "C" { struct llama_rs_parsed_chat; typedef struct llama_rs_parsed_chat * llama_rs_parsed_chat_handle; -typedef enum llama_rs_parsed_chat_free_status { - LLAMA_RS_PARSED_CHAT_FREE_OK = 0, - LLAMA_RS_PARSED_CHAT_FREE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_PARSED_CHAT_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, -} llama_rs_parsed_chat_free_status; - -llama_rs_parsed_chat_free_status llama_rs_parsed_chat_free( - llama_rs_parsed_chat_handle handle, - char ** out_error); +void llama_rs_parsed_chat_free(llama_rs_parsed_chat_handle handle); const char * llama_rs_parsed_chat_content(llama_rs_parsed_chat_handle handle, size_t * out_length); diff --git a/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs b/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs index e10969aa2..7b05c55f9 100644 --- a/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs +++ b/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs @@ -1,4 +1,9 @@ use anyhow::Result; +use llama_cpp_bindings::ChatAssistantTurn; +use llama_cpp_bindings::ChatSession; +use llama_cpp_bindings::ChatSessionOptions; +use llama_cpp_bindings::ChatToolChoice; +use llama_cpp_bindings::model::LlamaChatMessage; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -107,3 +112,55 @@ fn chat_session_renders_long_messages_in_full(fixture: &LlamaFixture<'_>) -> Res ); Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_generation_prompt_opens_the_assistant_turn_with_thinking( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, "hello", "[]", true)?; + + assert_eq!( + session.generation_prompt()?, + "<|im_start|>assistant\n\n" + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_omitted_assistant_turn_renders_only_the_messages( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = ChatSession::new( + fixture.model, + &[LlamaChatMessage::new( + "user".to_owned(), + "hello".to_owned(), + )?], + "[]", + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::Omitted, + enable_thinking: true, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + )?; + + assert_eq!(session.prompt()?, "<|im_start|>user\nhello<|im_end|>\n"); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/kv_cache_sequence_edits.rs b/llama-cpp-bindings-tests/tests/kv_cache_sequence_edits.rs index 2c9a5c156..281a4a27e 100644 --- a/llama-cpp-bindings-tests/tests/kv_cache_sequence_edits.rs +++ b/llama-cpp-bindings-tests/tests/kv_cache_sequence_edits.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use llama_cpp_bindings::ClearKvCacheSeqError; use llama_cpp_bindings_tests::prime_kv_cache::prime_kv_cache; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -131,3 +132,30 @@ fn copy_kv_cache_seq_accepts_an_explicit_range(fixture: &LlamaFixture<'_>) -> Re Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_recurrent_memory_refuses_to_remove_part_of_a_sequence( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_kv_cache(fixture, &mut context)?; + + assert_eq!( + context.clear_kv_cache_seq(Some(0), Some(1), Some(2)), + Err(ClearKvCacheSeqError::PartialSequenceNotRemoved { + seq_id: 0, + p0: 1, + p1: 2, + }) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/main.rs b/llama-cpp-bindings-tests/tests/main.rs index c0f25892e..39439a817 100644 --- a/llama-cpp-bindings-tests/tests/main.rs +++ b/llama-cpp-bindings-tests/tests/main.rs @@ -77,6 +77,7 @@ mod token_candidates; mod tokenization; mod tokenize_synthetic; mod tool_call_classification; +mod tool_call_grammar; mod tool_call_parsing; mod unconstrained_token_sampling; mod vocabulary_bounds; diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs index 7e9a9915b..a26972754 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs @@ -19,6 +19,7 @@ use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; use llama_cpp_bindings_tests::media_question_chat_session::media_question_chat_session; +use llama_cpp_bindings_tests::prime_kv_cache_with::prime_kv_cache_with; use llama_cpp_bindings_tests::prompt_question::PROMPT_QUESTION; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -676,3 +677,56 @@ fn gemma3_classifier_leaves_the_prompt_untouched_when_a_non_causal_image_exceeds Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 256, + n_ubatch = 256, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn classifier_reports_an_image_the_context_has_no_room_left_for( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; + let breakdown = ChunkTokenBreakdown::from_chunks(&chunks)?; + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + prime_kv_cache_with(fixture, &mut llama_ctx, &"hello ".repeat(230))?; + + let primed_positions = llama_ctx.kv_cache_seq_pos_max(0)? + 1; + let mut classifier = SampledTokenClassifier::undeterminable(fixture.model); + + assert!( + u64::from(llama_ctx.n_ctx()) - u64::try_from(primed_positions)? < breakdown.image, + "the primed context must have less room left than the image needs" + ); + assert!(matches!( + classifier.eval_multimodal_chunks( + &chunks, + mtmd_ctx, + &mut llama_ctx, + EvalMultimodalChunksParams { + start_position: primed_positions, + seq_id: 0, + n_batch: 256, + logits_last: true, + }, + ), + Err(EvalMultimodalChunksError::EvalFailed( + MtmdEvalError::EvalFailed { code: 1 } + )) + )); + assert_eq!(classifier.usage().input_image_tokens, 0); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs b/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs index 90c04803c..8b6c572f3 100644 --- a/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs +++ b/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs @@ -1,6 +1,7 @@ use anyhow::Context as _; use anyhow::Result; use llama_cpp_bindings::SampledTokenClassifier; +use llama_cpp_bindings::TokenUsageError; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_test_harness::LlamaFixture; @@ -252,3 +253,25 @@ fn discard_pending_prompt_tokens_clears_count_without_recording_usage( Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn cached_prompt_tokens_cannot_exceed_the_recorded_prompt(fixture: &LlamaFixture<'_>) { + let mut classifier = SampledTokenClassifier::undeterminable(fixture.model); + + classifier.record_prompt_tokens(2); + + assert_eq!( + classifier.record_cached_prompt_tokens(3), + Err(TokenUsageError::CachedExceedsPrompt { + cached_after: 3, + prompt: 2, + }) + ); +} diff --git a/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs b/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs index daacfd9fa..0e30ee091 100644 --- a/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs +++ b/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs @@ -3,13 +3,17 @@ use anyhow::Result; use anyhow::bail; use llama_cpp_bindings::BareJsonToolCalls; use llama_cpp_bindings::GenerationProgress; +use llama_cpp_bindings::IngestError; use llama_cpp_bindings::SampledToken; use llama_cpp_bindings::SampledTokenClassifier; +use llama_cpp_bindings::TokenOutsideVocabulary; use llama_cpp_bindings::TokenPiece; +use llama_cpp_bindings::TokenToStringError; use llama_cpp_bindings::ingest_outcome::IngestOutcome; use llama_cpp_bindings::model::AddBos; use llama_cpp_bindings::model::ParseSpecialTokens; use llama_cpp_bindings::sampled_token_section::SampledTokenSection; +use llama_cpp_bindings::token::LlamaToken; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -203,3 +207,39 @@ fn ingest_counts_every_token_through_the_end_of_generation( assert_eq!(classifier.usage().undeterminable_tokens, 2); Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn ingesting_a_token_outside_the_vocabulary_reports_it_in_every_classifier( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let token = LlamaToken::new(model.n_vocab()); + let session = user_turn_chat_session(model, "hello", "[]", false)?; + let mut outcomes = Vec::new(); + let expected = Err(IngestError::Detokenize( + TokenToStringError::TokenOutsideVocabulary(TokenOutsideVocabulary { + token, + n_vocab: model.n_vocab(), + }), + )); + + assert_eq!( + SampledTokenClassifier::undeterminable(model).ingest(token, &mut outcomes), + expected + ); + assert_eq!( + SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect) + .ingest(token, &mut outcomes), + expected + ); + assert_eq!(outcomes, []); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/tool_call_grammar.rs b/llama-cpp-bindings-tests/tests/tool_call_grammar.rs new file mode 100644 index 000000000..fb1b389b1 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/tool_call_grammar.rs @@ -0,0 +1,136 @@ +use anyhow::Context as _; +use anyhow::Result; +use llama_cpp_bindings::ChatAssistantTurn; +use llama_cpp_bindings::ChatOutputCompleteness; +use llama_cpp_bindings::ChatSession; +use llama_cpp_bindings::ChatSessionOptions; +use llama_cpp_bindings::ChatToolChoice; +use llama_cpp_bindings::model::LlamaChatMessage; +use llama_cpp_bindings::model::LlamaModel; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings_tests::chat_answer_generation::ChatAnswerGeneration; +use llama_cpp_bindings_tests::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; +use llama_cpp_bindings_tests::paris_weather_tool_call::paris_weather_tool_call; +use llama_cpp_bindings_tests::recognized_answer::recognized_answer; +use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +const MAX_TOOL_CALL_TOKENS: i32 = 128; + +fn weather_session_with_tool_choice( + model: &LlamaModel, + tool_choice: ChatToolChoice, +) -> Result> { + Ok(ChatSession::new( + model, + &[LlamaChatMessage::new( + "user".to_owned(), + "What is the weather in Paris?".to_owned(), + )?], + GET_WEATHER_TOOLS_JSON, + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::New, + enable_thinking: false, + parallel_tool_calls: false, + tool_choice, + }, + )?) +} + +fn generated_tool_call_answer( + fixture: &LlamaFixture<'_>, + session: &ChatSession<'_>, +) -> Result { + let grammar_sampler = session + .grammar_sampler()? + .context("a tool request must constrain generation with a grammar")?; + let mut sampler = LlamaSampler::chain_simple([grammar_sampler, LlamaSampler::greedy()?])?; + let answer = ChatAnswerGeneration { + fixture, + session, + sampler: &mut sampler, + max_generated_tokens: MAX_TOOL_CALL_TOKENS, + } + .run()?; + + Ok(answer.outcome.generated.text().to_owned()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn a_chat_session_without_tools_needs_no_grammar_sampler(fixture: &LlamaFixture<'_>) -> Result<()> { + let session = user_turn_chat_session(fixture.model, "Hello!", "[]", false)?; + + assert!(session.grammar_sampler()?.is_none()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn a_tool_choice_of_none_needs_no_grammar_sampler(fixture: &LlamaFixture<'_>) -> Result<()> { + let session = weather_session_with_tool_choice(fixture.model, ChatToolChoice::None)?; + + assert!(session.grammar_sampler()?.is_none()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn qwen36_required_tool_call_grammar_constrains_the_answer_to_a_tool_call( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = weather_session_with_tool_choice(fixture.model, ChatToolChoice::Required)?; + let answer = generated_tool_call_answer(fixture, &session)?; + + assert!(!session.grammar_is_lazy()); + assert_eq!( + recognized_answer(&session, &answer, ChatOutputCompleteness::Finished)?.tool_calls, + [paris_weather_tool_call()] + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn qwen36_automatic_tool_call_grammar_constrains_the_tool_call_after_its_trigger( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = weather_session_with_tool_choice(fixture.model, ChatToolChoice::Auto)?; + let answer = generated_tool_call_answer(fixture, &session)?; + + assert!(session.grammar_is_lazy()); + assert_eq!( + recognized_answer(&session, &answer, ChatOutputCompleteness::Finished)?.tool_calls, + [paris_weather_tool_call()] + ); + + Ok(()) +} diff --git a/llama-cpp-bindings/src/bare_json_tool_call_probe.rs b/llama-cpp-bindings/src/bare_json_tool_call_probe.rs index b7b5bd348..ad75957f8 100644 --- a/llama-cpp-bindings/src/bare_json_tool_call_probe.rs +++ b/llama-cpp-bindings/src/bare_json_tool_call_probe.rs @@ -41,44 +41,48 @@ impl BareJsonToolCallProbe { pub fn pass(&mut self, outcome: IngestOutcome, released: &mut Vec) { let is_content = is_visible_content(&outcome); - match &mut self.state { - ProbeState::Holding { .. } if !is_content => { - self.release(released); + match mem::replace(&mut self.state, ProbeState::Idle) { + ProbeState::Holding { held, .. } if !is_content => { + released.extend(held); released.push(outcome); } - ProbeState::Holding { probe, held } => { + ProbeState::Holding { + mut probe, + mut held, + } => { let probe_outcome = probe.feed(outcome.piece.raw()); + held.push(outcome); - self.settle(probe_outcome, released); + self.settle(probe_outcome, probe, held, released); } ProbeState::Idle if is_content && outcome.piece.raw().trim_start().starts_with('{') => { let mut probe = StreamingJsonProbe::default(); let probe_outcome = probe.feed(outcome.piece.raw()); - self.state = ProbeState::Holding { - probe, - held: vec![outcome], - }; - self.settle(probe_outcome, released); + self.settle(probe_outcome, probe, vec![outcome], released); } ProbeState::Idle => released.push(outcome), } } - fn settle(&mut self, probe_outcome: JsonProbeOutcome, released: &mut Vec) { + fn settle( + &mut self, + probe_outcome: JsonProbeOutcome, + probe: StreamingJsonProbe, + held: Vec, + released: &mut Vec, + ) { match probe_outcome { - JsonProbeOutcome::StillPossiblyValid => {} - JsonProbeOutcome::CompletedValid => self.commit_as_tool_call(released), - JsonProbeOutcome::Failed => self.release(released), - } - } - - fn commit_as_tool_call(&mut self, released: &mut Vec) { - if let ProbeState::Holding { held, .. } = mem::replace(&mut self.state, ProbeState::Idle) { - released.extend(held.into_iter().map(|outcome| IngestOutcome { - sampled_token: SampledToken::ToolCall(outcome.sampled_token.token()), - piece: outcome.piece, - })); + JsonProbeOutcome::StillPossiblyValid => { + self.state = ProbeState::Holding { probe, held }; + } + JsonProbeOutcome::CompletedValid => { + released.extend(held.into_iter().map(|outcome| IngestOutcome { + sampled_token: SampledToken::ToolCall(outcome.sampled_token.token()), + piece: outcome.piece, + })); + } + JsonProbeOutcome::Failed => released.extend(held), } } @@ -207,4 +211,13 @@ mod tests { ] ); } + + #[test] + fn releasing_an_idle_probe_releases_nothing() { + let mut released = Vec::new(); + + BareJsonToolCallProbe::default().release(&mut released); + + assert_eq!(released, []); + } } diff --git a/llama-cpp-bindings/src/chat_classification.rs b/llama-cpp-bindings/src/chat_classification.rs index 9247e210b..ab951c9b1 100644 --- a/llama-cpp-bindings/src/chat_classification.rs +++ b/llama-cpp-bindings/src/chat_classification.rs @@ -1,7 +1,54 @@ +use std::ops::Range; + +use crate::chat_output_completeness::ChatOutputCompleteness; +use crate::chat_piece_role::ChatPieceRole; use crate::chat_spans::ChatSpans; +use crate::sampled_token_section::SampledTokenSection; #[derive(Clone, Debug, Eq, PartialEq)] pub enum ChatClassification { Spans(ChatSpans), - OutputDoesNotMatchFormat, + OutputDoesNotMatchFormat(ChatOutputCompleteness), +} + +impl ChatClassification { + /// Returns how the token whose text occupies `bytes` of the output is attributed, or `None` + /// while the classification cannot settle it yet. An answer that does not match the + /// template's format attributes no token to a section once it is finished. + #[must_use] + pub fn classify_token(&self, bytes: &Range) -> Option { + match self { + Self::Spans(spans) => spans.classify_token(bytes), + Self::OutputDoesNotMatchFormat(ChatOutputCompleteness::Partial) => None, + Self::OutputDoesNotMatchFormat(ChatOutputCompleteness::Finished) => { + Some(ChatPieceRole::Text(SampledTokenSection::Pending)) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::ChatClassification; + use crate::chat_output_completeness::ChatOutputCompleteness; + use crate::chat_piece_role::ChatPieceRole; + use crate::sampled_token_section::SampledTokenSection; + + #[test] + fn a_partial_answer_that_does_not_match_the_format_settles_no_token() { + assert_eq!( + ChatClassification::OutputDoesNotMatchFormat(ChatOutputCompleteness::Partial) + .classify_token(&(0..3)), + None + ); + } + + #[test] + fn a_finished_answer_that_does_not_match_the_format_is_visible_text_of_no_section() { + assert_eq!( + ChatClassification::OutputDoesNotMatchFormat(ChatOutputCompleteness::Finished) + .classify_token(&(0..3)), + Some(ChatPieceRole::Text(SampledTokenSection::Pending)) + ); + } } diff --git a/llama-cpp-bindings/src/chat_grammar_trigger.rs b/llama-cpp-bindings/src/chat_grammar_trigger.rs index 51a3e6d3c..a58fbfe2e 100644 --- a/llama-cpp-bindings/src/chat_grammar_trigger.rs +++ b/llama-cpp-bindings/src/chat_grammar_trigger.rs @@ -7,3 +7,80 @@ pub enum ChatGrammarTrigger { Pattern(String), PatternFull(String), } + +impl ChatGrammarTrigger { + /// Builds the trigger the wrapper reports as `trigger_type` with `value`, reading its token + /// only for a token trigger. + /// + /// # Errors + /// + /// Returns [`crate::FfiStatusError`] for a trigger type unknown to this binding. + pub fn from_wrapper( + trigger_type: llama_cpp_bindings_sys::llama_rs_chat_grammar_trigger_type, + value: String, + token: impl FnOnce() -> LlamaToken, + ) -> Result { + match trigger_type { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_TOKEN => { + Ok(Self::Token(token())) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_WORD => { + Ok(Self::Word(value)) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN => { + Ok(Self::Pattern(value)) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL => { + Ok(Self::PatternFull(value)) + } + other => Err(crate::FfiStatusError { + operation: "llama_rs_chat_session_grammar_trigger_type", + code: i64::from(other), + }), + } + } +} + +#[cfg(test)] +mod tests { + use super::ChatGrammarTrigger; + use crate::token::LlamaToken; + + fn trigger_of( + trigger_type: llama_cpp_bindings_sys::llama_rs_chat_grammar_trigger_type, + ) -> Result { + ChatGrammarTrigger::from_wrapper(trigger_type, "".to_owned(), || { + LlamaToken::new(151_657) + }) + } + + #[test] + fn every_wrapper_trigger_type_maps_to_its_trigger() { + assert_eq!( + [ + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_TOKEN, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_WORD, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL, + ] + .map(trigger_of), + [ + Ok(ChatGrammarTrigger::Token(LlamaToken::new(151_657))), + Ok(ChatGrammarTrigger::Word("".to_owned())), + Ok(ChatGrammarTrigger::Pattern("".to_owned())), + Ok(ChatGrammarTrigger::PatternFull("".to_owned())), + ] + ); + } + + #[test] + fn an_unknown_wrapper_trigger_type_is_a_status_error() { + assert_eq!( + trigger_of(99), + Err(crate::FfiStatusError { + operation: "llama_rs_chat_session_grammar_trigger_type", + code: 99, + }) + ); + } +} diff --git a/llama-cpp-bindings/src/chat_message_parse_outcome.rs b/llama-cpp-bindings/src/chat_message_parse_outcome.rs index 845a90e39..702d0f0a8 100644 --- a/llama-cpp-bindings/src/chat_message_parse_outcome.rs +++ b/llama-cpp-bindings/src/chat_message_parse_outcome.rs @@ -1,4 +1,5 @@ use llama_cpp_bindings_types::ParsedChatMessage; +use llama_cpp_bindings_types::ParsedToolCall; use crate::raw_chat_message::RawChatMessage; @@ -7,3 +8,72 @@ pub enum ChatMessageParseOutcome { Recognized(ParsedChatMessage), Unrecognized(RawChatMessage), } + +impl ChatMessageParseOutcome { + /// The message this answer becomes when `tool_calls` were recognized after it: a recognized + /// answer keeps its content and reasoning, an unrecognized one is kept whole as content. + #[must_use] + pub fn followed_by_tool_calls(self, tool_calls: Vec) -> ParsedChatMessage { + match self { + Self::Recognized(message) => { + ParsedChatMessage::new(message.content, message.reasoning_content, tool_calls) + } + Self::Unrecognized(raw) => ParsedChatMessage::new(raw.text, String::new(), tool_calls), + } + } +} + +#[cfg(test)] +mod tests { + use llama_cpp_bindings_types::ParsedChatMessage; + use llama_cpp_bindings_types::ParsedToolCall; + use llama_cpp_bindings_types::ParsedToolCallId; + use llama_cpp_bindings_types::ToolCallArguments; + use serde_json::json; + + use super::ChatMessageParseOutcome; + use crate::chat_output_completeness::ChatOutputCompleteness; + use crate::raw_chat_message::RawChatMessage; + + fn weather_call() -> ParsedToolCall { + ParsedToolCall::new( + ParsedToolCallId::NotProvidedByModel, + "get_weather".to_owned(), + ToolCallArguments::ValidJson(json!({"location": "Paris"})), + ) + } + + #[test] + fn a_recognized_answer_keeps_its_content_and_reasoning_before_the_tool_calls() { + assert_eq!( + ChatMessageParseOutcome::Recognized(ParsedChatMessage::new( + "Checking.".to_owned(), + "The user wants the weather.".to_owned(), + Vec::new(), + )) + .followed_by_tool_calls(vec![weather_call()]), + ParsedChatMessage::new( + "Checking.".to_owned(), + "The user wants the weather.".to_owned(), + vec![weather_call()], + ) + ); + } + + #[test] + fn an_unrecognized_answer_is_kept_whole_as_content_before_the_tool_calls() { + assert_eq!( + ChatMessageParseOutcome::Unrecognized(RawChatMessage { + text: "Checking.".to_owned(), + completeness: ChatOutputCompleteness::Finished, + ffi_error_message: "the format differs".to_owned(), + }) + .followed_by_tool_calls(vec![weather_call()]), + ParsedChatMessage::new( + "Checking.".to_owned(), + String::new(), + vec![weather_call()] + ) + ); + } +} diff --git a/llama-cpp-bindings/src/chat_session.rs b/llama-cpp-bindings/src/chat_session.rs index cee8ed08e..b0475108b 100644 --- a/llama-cpp-bindings/src/chat_session.rs +++ b/llama-cpp-bindings/src/chat_session.rs @@ -23,15 +23,14 @@ use crate::chat_spans::ChatSpans; use crate::chat_template_tool_calls::known_marker_candidates; use crate::cpp_text_view::cpp_text_view; use crate::error::chat_session_error::ChatSessionError; -use crate::error::grammar_error::GrammarError; use crate::error::parse_chat_message_error::ParseChatMessageError; +use crate::lazy_grammar_triggers::LazyGrammarTriggers; use crate::model::AddBos; use crate::model::LlamaModel; use crate::model::ParseSpecialTokens; use crate::model::llama_chat_message::LlamaChatMessage; use crate::parsed_chat::take_parsed_chat_message; use crate::raw_chat_message::RawChatMessage; -use crate::result_after_release::result_after_release; use crate::sampling::LlamaSampler; use crate::token::LlamaToken; use crate::tool_call_format::ToolCallFormatOutcome; @@ -85,17 +84,6 @@ fn session_text(text: *const c_char, length: usize) -> Result String { - if pattern.is_empty() { - return "^$".to_owned(); - } - - let start = if pattern.starts_with('^') { "" } else { "^" }; - let end = if pattern.ends_with('$') { "" } else { "$" }; - - format!("{start}{pattern}{end}") -} - const fn contract_violation( operation: &'static str, detail: &'static str, @@ -187,38 +175,6 @@ fn chat_session_create_status_to_result( } } -fn chat_session_free_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_chat_session_free_status, - out_error: *mut c_char, -) -> Result<(), ChatSessionError> { - const OPERATION: &str = "llama_rs_chat_session_free"; - - match status { - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_FREE_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_FREE_ERROR_STRING_ALLOCATION_FAILED => { - Err(ChatSessionError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_FREE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(ChatSessionError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - out_error, - OPERATION, - "reported a thrown C++ exception without an error message", - ) - }?; - Err(ChatSessionError::Reported { message }) - } - other => Err(crate::FfiStatusError { - operation: OPERATION, - code: i64::from(other), - } - .into()), - } -} - fn chat_session_parse_status_to_result( status: llama_cpp_bindings_sys::llama_rs_chat_session_parse_status, out_handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, @@ -283,39 +239,6 @@ fn chat_session_parse_status_to_result( } } -fn chat_spans_free_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_chat_spans_free_status, - free_error: *mut c_char, -) -> Result<(), ParseChatMessageError> { - const OPERATION: &str = "llama_rs_chat_spans_free"; - - match status { - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPANS_FREE_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPANS_FREE_ERROR_STRING_ALLOCATION_FAILED => { - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPANS_FREE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPANS_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION => { - Err(ParseChatMessageError::DestructorFailed { - message: unsafe { - read_and_free_cpp_string( - free_error, - OPERATION, - "reported a thrown C++ exception without an error message", - ) - }?, - }) - } - other => Err(crate::FfiStatusError { - operation: OPERATION, - code: i64::from(other), - } - .into()), - } -} - fn read_chat_spans( spans: NonNull, completeness: ChatOutputCompleteness, @@ -323,36 +246,29 @@ fn read_chat_spans( let count = unsafe { llama_cpp_bindings_sys::llama_rs_chat_spans_count(spans.as_ptr()) }; let read = (0..count) .map(|index| { - let kind = ChatSpanKind::try_from(unsafe { + let bytes = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_spans_start(spans.as_ptr(), index) + ..llama_cpp_bindings_sys::llama_rs_chat_spans_end(spans.as_ptr(), index) + }; + + ChatSpanKind::try_from(unsafe { llama_cpp_bindings_sys::llama_rs_chat_spans_kind(spans.as_ptr(), index) - })?; - let start = - unsafe { llama_cpp_bindings_sys::llama_rs_chat_spans_start(spans.as_ptr(), index) }; - let end = - unsafe { llama_cpp_bindings_sys::llama_rs_chat_spans_end(spans.as_ptr(), index) }; - - Ok(ChatSpan { - kind, - bytes: start..end, }) + .map(|kind| ChatSpan { kind, bytes }) }) - .collect::, ParseChatMessageError>>() + .collect::, crate::FfiStatusError>>() .map(|spans_read| ChatSpans { parsed_length: unsafe { llama_cpp_bindings_sys::llama_rs_chat_spans_parsed_length(spans.as_ptr()) }, spans: spans_read, completeness, - }); - let mut free_error: *mut c_char = ptr::null_mut(); - let free_status = unsafe { - llama_cpp_bindings_sys::llama_rs_chat_spans_free(spans.as_ptr(), &raw mut free_error) - }; + }) + .map_err(ParseChatMessageError::from); + + unsafe { llama_cpp_bindings_sys::llama_rs_chat_spans_free(spans.as_ptr()) }; - result_after_release( - read, - chat_spans_free_status_to_result(free_status, free_error), - ) + read } fn chat_session_classify_status_to_result( @@ -371,7 +287,7 @@ fn chat_session_classify_status_to_result( .and_then(|spans| read_chat_spans(spans, completeness)) .map(ChatClassification::Spans), llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_OUTPUT_DOES_NOT_MATCH_FORMAT => { - Ok(ChatClassification::OutputDoesNotMatchFormat) + Ok(ChatClassification::OutputDoesNotMatchFormat(completeness)) } llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_THREW_CXX_EXCEPTION => { Err(ParseChatMessageError::Reported { @@ -413,70 +329,6 @@ fn chat_session_classify_status_to_result( } } -fn regex_escape_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_regex_escape_status, - out_string: *mut c_char, - out_error: *mut c_char, -) -> Result { - const OPERATION: &str = "llama_rs_regex_escape"; - - match status { - llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_OK => Ok(unsafe { - read_and_free_cpp_string( - out_string, - OPERATION, - "success status contained a null string", - ) - }?), - llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_THREW_CXX_EXCEPTION => { - Err(GrammarError::Reported { - message: unsafe { - read_and_free_cpp_string( - out_error, - OPERATION, - "reported a thrown C++ exception without an error message", - ) - }?, - }) - } - llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_ERROR_STRING_ALLOCATION_FAILED => { - Err(GrammarError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(GrammarError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_TEXT_ARG => { - Err(contract_violation(OPERATION, "was given a null text argument").into()) - } - llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_OUT_STRING_ARG => { - Err(contract_violation(OPERATION, "was given a null out_string argument").into()) - } - llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_OUT_ERROR_ARG => { - Err(contract_violation(OPERATION, "was given a null out_error argument").into()) - } - other => Err(crate::FfiStatusError { - operation: OPERATION, - code: i64::from(other), - } - .into()), - } -} - -fn regex_escaped(word: &str) -> Result { - let mut out_string: *mut c_char = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_regex_escape( - word.as_ptr().cast::(), - word.len(), - &raw mut out_string, - &raw mut out_error, - ) - }; - - regex_escape_status_to_result(status, out_string, out_error) -} - /// One chat request rendered by the model's chat template the way llama-server renders it: the /// prompt, the grammar that constrains tool calls, and the parser of the model's answer. pub struct ChatSession<'model> { @@ -692,30 +544,18 @@ impl<'model> ChatSession<'model> { ) })?; - match trigger_type { - llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_TOKEN => { - Ok(ChatGrammarTrigger::Token(LlamaToken::new(unsafe { + Ok(ChatGrammarTrigger::from_wrapper( + trigger_type, + value, + || { + LlamaToken::new(unsafe { llama_cpp_bindings_sys::llama_rs_chat_session_grammar_trigger_token( self.session.as_ptr(), index, ) - }))) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_WORD => { - Ok(ChatGrammarTrigger::Word(value)) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN => { - Ok(ChatGrammarTrigger::Pattern(value)) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL => { - Ok(ChatGrammarTrigger::PatternFull(value)) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_chat_session_grammar_trigger_type", - code: i64::from(other), - } - .into()), - } + }) + }, + )?) }) .collect() } @@ -767,24 +607,6 @@ impl<'model> ChatSession<'model> { ) } - /// # Errors - /// - /// Returns [`ChatSessionError::SessionTextNotUtf8`] when a stop is not UTF-8. - pub fn additional_stops(&self) -> Result, ChatSessionError> { - self.texts_of( - unsafe { - llama_cpp_bindings_sys::llama_rs_chat_session_additional_stop_count( - self.session.as_ptr(), - ) - }, - |session, index, out_length| unsafe { - llama_cpp_bindings_sys::llama_rs_chat_session_additional_stop( - session, index, out_length, - ) - }, - ) - } - /// Whether the template this request rendered with can render tool calls. When it cannot, /// [`Self::parse`] recognizes tool calls in the formats other templates use. #[must_use] @@ -822,26 +644,15 @@ impl<'model> ChatSession<'model> { return Ok(Some(sampler)); } - let mut trigger_patterns = Vec::new(); - let mut trigger_tokens = Vec::new(); - - for trigger in self.grammar_triggers()? { - match trigger { - ChatGrammarTrigger::Token(token) => trigger_tokens.push(token), - ChatGrammarTrigger::Word(word) => trigger_patterns.push(regex_escaped(&word)?), - ChatGrammarTrigger::Pattern(pattern) => trigger_patterns.push(pattern), - ChatGrammarTrigger::PatternFull(pattern) => { - trigger_patterns.push(anchored_pattern(&pattern)); - } - } - } + let LazyGrammarTriggers { patterns, tokens } = + LazyGrammarTriggers::of(self.grammar_triggers()?)?; Ok(Some(LlamaSampler::grammar_lazy( self.model, &grammar, GRAMMAR_ROOT, - &trigger_patterns, - &trigger_tokens, + &patterns, + &tokens, )?)) } @@ -881,18 +692,8 @@ impl<'model> ChatSession<'model> { .map_or_else(|| output.text(), |open| &output.text()[..open]); return Ok(Some( - match self - .parse_with_template(&ChatOutput::from(before_tool_calls), completeness)? - { - ChatMessageParseOutcome::Recognized(prefix) => ParsedChatMessage::new( - prefix.content, - prefix.reasoning_content, - tool_calls, - ), - ChatMessageParseOutcome::Unrecognized(unparsed_prefix) => { - ParsedChatMessage::new(unparsed_prefix.text, String::new(), tool_calls) - } - }, + self.parse_with_template(&ChatOutput::from(before_tool_calls), completeness)? + .followed_by_tool_calls(tool_calls), )); } } @@ -952,36 +753,21 @@ impl<'model> ChatSession<'model> { impl Drop for ChatSession<'_> { fn drop(&mut self) { - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_chat_session_free( - self.session.as_ptr(), - &raw mut out_error, - ) - }; - - if let Err(destructor_failure) = chat_session_free_status_to_result(status, out_error) { - log::error!("{destructor_failure}"); - } + unsafe { llama_cpp_bindings_sys::llama_rs_chat_session_free(self.session.as_ptr()) }; } } #[cfg(test)] mod tests { use std::ffi::CStr; - use std::ffi::c_char; use std::ptr; use serde_json::json; - use super::anchored_pattern; use super::chat_session_classify_status_to_result; use super::chat_session_create_status_to_result; - use super::chat_session_free_status_to_result; use super::chat_session_parse_status_to_result; - use super::chat_spans_free_status_to_result; use super::messages_json; - use super::regex_escape_status_to_result; use crate::chat_assistant_prefill::ChatAssistantPrefill; use crate::chat_assistant_turn::ChatAssistantTurn; use crate::chat_classification::ChatClassification; @@ -989,7 +775,6 @@ mod tests { use crate::chat_output::ChatOutput; use crate::chat_output_completeness::ChatOutputCompleteness; use crate::error::chat_session_error::ChatSessionError; - use crate::error::grammar_error::GrammarError; use crate::error::parse_chat_message_error::ParseChatMessageError; use crate::model::llama_chat_message::LlamaChatMessage; use crate::raw_chat_message::RawChatMessage; @@ -999,21 +784,6 @@ mod tests { fn(String) -> ChatSessionError, ); - fn cpp_message(message: &CStr) -> *mut c_char { - unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(message.as_ptr()) } - } - - fn contract_violation( - operation: &'static str, - detail: &'static str, - ) -> crate::FfiContractError { - crate::FfiContractError { operation, detail } - } - - fn status_error(operation: &'static str, code: i64) -> crate::FfiStatusError { - crate::FfiStatusError { operation, code } - } - fn parsed_messages(messages_json: &CStr) -> serde_json::Value { serde_json::from_str(messages_json.to_str().unwrap()).unwrap() } @@ -1053,10 +823,10 @@ mod tests { assert_eq!( chat_session_create_status_to_result(status, ptr::null_mut(), ptr::null_mut()) .err(), - Some(ChatSessionError::FfiContract(contract_violation( - "llama_rs_chat_session_create", + Some(ChatSessionError::FfiContract(crate::FfiContractError { + operation: "llama_rs_chat_session_create", detail - ))) + })) ); } } @@ -1092,11 +862,9 @@ mod tests { for (status, expected_error) in refusals { assert_eq!( - chat_session_create_status_to_result( - status, - ptr::null_mut(), - cpp_message(c"the reason") - ) + chat_session_create_status_to_result(status, ptr::null_mut(), unsafe { + llama_cpp_bindings_sys::llama_rs_string_dup(c"the reason".as_ptr()) + }) .err(), Some(expected_error("the reason".to_owned())) ); @@ -1112,10 +880,10 @@ mod tests { ptr::null_mut() ) .err(), - Some(ChatSessionError::FfiContract(contract_violation( - "llama_rs_chat_session_create", - "reported a failure without an error message" - ))) + Some(ChatSessionError::FfiContract(crate::FfiContractError { + operation: "llama_rs_chat_session_create", + detail: "reported a failure without an error message" + })) ); } @@ -1143,47 +911,10 @@ mod tests { fn creating_a_session_preserves_an_unknown_status() { assert_eq!( chat_session_create_status_to_result(255, ptr::null_mut(), ptr::null_mut()).err(), - Some(ChatSessionError::FfiStatus(status_error( - "llama_rs_chat_session_create", - 255 - ))) - ); - } - - #[test] - fn freeing_a_session_maps_every_status() { - assert_eq!( - [ - chat_session_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_FREE_OK, - ptr::null_mut() - ), - chat_session_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_FREE_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut() - ), - chat_session_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_FREE_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null_mut() - ), - chat_session_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - cpp_message(c"the destructor threw") - ), - chat_session_free_status_to_result(255, ptr::null_mut()), - ], - [ - Ok(()), - Err(ChatSessionError::NotEnoughMemory), - Err(ChatSessionError::LlamaCppOutOfMemory), - Err(ChatSessionError::Reported { - message: "the destructor threw".to_owned() - }), - Err(ChatSessionError::FfiStatus(status_error( - "llama_rs_chat_session_free", - 255 - ))), - ] + Some(ChatSessionError::FfiStatus(crate::FfiStatusError { + operation: "llama_rs_chat_session_create", + code: 255 + })) ); } @@ -1195,7 +926,9 @@ mod tests { chat_session_parse_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_OUTPUT_DOES_NOT_MATCH_FORMAT, ptr::null_mut(), - cpp_message(c"the format differs"), + unsafe { + llama_cpp_bindings_sys::llama_rs_string_dup(c"the format differs".as_ptr()) + }, &output, ChatOutputCompleteness::Finished, ), @@ -1221,10 +954,12 @@ mod tests { .err() }; let contract = |detail| { - Some(ParseChatMessageError::FfiContract(contract_violation( - "llama_rs_chat_session_parse", - detail, - ))) + Some(ParseChatMessageError::FfiContract( + crate::FfiContractError { + operation: "llama_rs_chat_session_parse", + detail, + }, + )) }; assert_eq!( @@ -1235,7 +970,7 @@ mod tests { ), parse( llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION, - cpp_message(c"the parser threw") + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"the parser threw".as_ptr()) } ), parse( llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_ERROR_STRING_ALLOCATION_FAILED, @@ -1279,10 +1014,7 @@ mod tests { contract("was given a null byte_tokens argument"), contract("was given a null out_handle argument"), contract("was given a null out_error argument"), - Some(ParseChatMessageError::FfiStatus(status_error( - "llama_rs_chat_session_parse", - 255 - ))), + Some(ParseChatMessageError::FfiStatus(crate::FfiStatusError { operation: "llama_rs_chat_session_parse", code: 255 })), ] ); } @@ -1296,7 +1028,9 @@ mod tests { ptr::null_mut(), ChatOutputCompleteness::Partial, ), - Ok(ChatClassification::OutputDoesNotMatchFormat) + Ok(ChatClassification::OutputDoesNotMatchFormat( + ChatOutputCompleteness::Partial + )) ); } @@ -1312,10 +1046,12 @@ mod tests { .err() }; let contract = |detail| { - Some(ParseChatMessageError::FfiContract(contract_violation( - "llama_rs_chat_session_classify", - detail, - ))) + Some(ParseChatMessageError::FfiContract( + crate::FfiContractError { + operation: "llama_rs_chat_session_classify", + detail, + }, + )) }; assert_eq!( @@ -1326,7 +1062,7 @@ mod tests { ), classify( llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_THREW_CXX_EXCEPTION, - cpp_message(c"the parser threw") + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"the parser threw".as_ptr()) } ), classify( llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_ERROR_STRING_ALLOCATION_FAILED, @@ -1370,134 +1106,11 @@ mod tests { contract("was given a null byte_tokens argument"), contract("was given a null out_spans argument"), contract("was given a null out_error argument"), - Some(ParseChatMessageError::FfiStatus(status_error( - "llama_rs_chat_session_classify", - 255 - ))), + Some(ParseChatMessageError::FfiStatus(crate::FfiStatusError { operation: "llama_rs_chat_session_classify", code: 255 })), ] ); } - #[test] - fn freeing_classified_spans_maps_every_status() { - assert_eq!( - [ - chat_spans_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPANS_FREE_OK, - ptr::null_mut() - ), - chat_spans_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPANS_FREE_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut() - ), - chat_spans_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPANS_FREE_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null_mut() - ), - chat_spans_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPANS_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - cpp_message(c"the destructor threw") - ), - chat_spans_free_status_to_result(255, ptr::null_mut()), - ], - [ - Ok(()), - Err(ParseChatMessageError::NotEnoughMemory), - Err(ParseChatMessageError::LlamaCppOutOfMemory), - Err(ParseChatMessageError::DestructorFailed { - message: "the destructor threw".to_owned() - }), - Err(ParseChatMessageError::FfiStatus(status_error( - "llama_rs_chat_spans_free", - 255 - ))), - ] - ); - } - - #[test] - fn escaping_a_trigger_word_maps_every_status() { - let escape = |status, out_string, out_error| { - regex_escape_status_to_result(status, out_string, out_error) - }; - let contract = |detail| { - Err(GrammarError::FfiContract(contract_violation( - "llama_rs_regex_escape", - detail, - ))) - }; - - assert_eq!( - [ - escape( - llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_OK, - cpp_message(c"\\[TOOL\\]"), - ptr::null_mut() - ), - escape( - llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_OK, - ptr::null_mut(), - ptr::null_mut() - ), - escape( - llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null_mut(), - cpp_message(c"the escape threw") - ), - escape( - llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - ptr::null_mut() - ), - escape( - llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null_mut(), - ptr::null_mut() - ), - escape( - llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_TEXT_ARG, - ptr::null_mut(), - ptr::null_mut() - ), - escape( - llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_OUT_STRING_ARG, - ptr::null_mut(), - ptr::null_mut() - ), - escape( - llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_OUT_ERROR_ARG, - ptr::null_mut(), - ptr::null_mut() - ), - escape(255, ptr::null_mut(), ptr::null_mut()), - ], - [ - Ok("\\[TOOL\\]".to_owned()), - contract("success status contained a null string"), - Err(GrammarError::Reported { - message: "the escape threw".to_owned() - }), - Err(GrammarError::NotEnoughMemory), - Err(GrammarError::LlamaCppOutOfMemory), - contract("was given a null text argument"), - contract("was given a null out_string argument"), - contract("was given a null out_error argument"), - Err(GrammarError::FfiStatus(status_error( - "llama_rs_regex_escape", - 255 - ))), - ] - ); - } - - #[test] - fn a_full_match_pattern_is_anchored_at_both_ends_once() { - assert_eq!( - ["", "abc", "^abc", "abc$", "^abc$"].map(anchored_pattern), - ["^$", "^abc$", "^abc$", "^abc$", "^abc$"].map(str::to_owned) - ); - } - #[test] fn a_new_assistant_turn_adds_no_message() { let messages = [LlamaChatMessage::new("user".to_owned(), "hi".to_owned()).unwrap()]; @@ -1535,4 +1148,39 @@ mod tests { json!([{"role": "assistant", "content": "The", "reasoning_content": "done"}]) ); } + + #[test] + fn a_parse_or_classification_failure_without_a_message_is_a_contract_violation() { + let output = ChatOutput::default(); + + assert_eq!( + [ + chat_session_parse_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + ptr::null_mut(), + &output, + ChatOutputCompleteness::Finished, + ) + .err(), + chat_session_classify_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + ptr::null_mut(), + ChatOutputCompleteness::Finished, + ) + .err(), + ], + [ + Some(ParseChatMessageError::FfiContract(crate::FfiContractError { + operation: "llama_rs_chat_session_parse", + detail: "reported a failure without an error message", + })), + Some(ParseChatMessageError::FfiContract(crate::FfiContractError { + operation: "llama_rs_chat_session_classify", + detail: "reported a thrown C++ exception without an error message", + })), + ] + ); + } } diff --git a/llama-cpp-bindings/src/chat_templates.rs b/llama-cpp-bindings/src/chat_templates.rs index 7d720337c..981d74493 100644 --- a/llama-cpp-bindings/src/chat_templates.rs +++ b/llama-cpp-bindings/src/chat_templates.rs @@ -69,36 +69,6 @@ fn chat_templates_init_status_to_result( } } -fn chat_templates_free_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_chat_templates_free_status, - out_error: *mut c_char, -) -> Result<(), ChatTemplatesError> { - match status { - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_FREE_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_FREE_ERROR_STRING_ALLOCATION_FAILED => { - Err(ChatTemplatesError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_FREE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(ChatTemplatesError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - out_error, - "llama_rs_chat_templates_free", - "reported a thrown C++ exception without an error message", - ) - }?; - Err(ChatTemplatesError::Reported { message }) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_chat_templates_free", - code: i64::from(other), - } - .into()), - } -} - /// The chat templates of a model as llama.cpp prepares them once per model: with the model's /// BOS and EOS, the template variant for tool calls, and llama.cpp's template fixes applied. #[derive(Debug)] @@ -149,34 +119,18 @@ impl ChatTemplates { impl Drop for ChatTemplates { fn drop(&mut self) { - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_chat_templates_free( - self.templates.as_ptr(), - &raw mut out_error, - ) - }; - - if let Err(destructor_failure) = chat_templates_free_status_to_result(status, out_error) { - log::error!("{destructor_failure}"); - } + unsafe { llama_cpp_bindings_sys::llama_rs_chat_templates_free(self.templates.as_ptr()) }; } } #[cfg(test)] mod tests { - use std::ffi::CStr; use std::ffi::c_char; use std::ptr; - use super::chat_templates_free_status_to_result; use super::chat_templates_init_status_to_result; use crate::error::chat_templates_error::ChatTemplatesError; - fn cpp_message(message: &CStr) -> *mut c_char { - unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(message.as_ptr()) } - } - fn init_error( status: llama_cpp_bindings_sys::llama_rs_chat_templates_init_status, out_error: *mut c_char, @@ -213,7 +167,7 @@ mod tests { ), init_error( llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, - cpp_message(c"the template threw") + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"the template threw".as_ptr()) } ), init_error( llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_MODEL_ARG, @@ -249,39 +203,15 @@ mod tests { } #[test] - fn releasing_templates_maps_every_status() { + fn a_preparation_exception_without_a_message_is_a_contract_violation() { assert_eq!( - [ - chat_templates_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_FREE_OK, - ptr::null_mut() - ), - chat_templates_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_FREE_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut() - ), - chat_templates_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_FREE_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null_mut() - ), - chat_templates_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - cpp_message(c"the destructor threw") - ), - chat_templates_free_status_to_result(255, ptr::null_mut()), - ], - [ - Ok(()), - Err(ChatTemplatesError::NotEnoughMemory), - Err(ChatTemplatesError::LlamaCppOutOfMemory), - Err(ChatTemplatesError::Reported { - message: "the destructor threw".to_owned() - }), - Err(ChatTemplatesError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_chat_templates_free", - code: 255, - })), - ] + init_error( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut() + ), + Some(contract( + "reported a thrown C++ exception without an error message" + )) ); } } diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index 58089f7da..e9f9ca80b 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -1268,6 +1268,25 @@ mod new_context_with_model_status_to_result_tests { assert_eq!(result, Err(LlamaContextLoadError::NotEnoughMemory)); } + #[test] + fn new_context_cxx_exception_reports_its_message() { + let out_error = unsafe { + llama_cpp_bindings_sys::llama_rs_string_dup(c"context construction threw".as_ptr()) + }; + let result = new_context_with_model_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + out_error, + ); + + assert_eq!( + result, + Err(LlamaContextLoadError::Reported { + message: "context construction threw".to_owned() + }) + ); + } + #[test] fn new_context_cxx_exception_without_a_message_is_a_contract_error() { let result = new_context_with_model_status_to_result( @@ -1370,6 +1389,237 @@ mod new_context_with_model_status_to_result_tests { } } +#[cfg(test)] +mod batch_ext_init_status_to_result_tests { + use std::ptr; + + use super::batch_ext_init_status_to_result; + use crate::LlamaContextLoadError; + + #[test] + fn initializing_a_batch_reports_each_null_pointer_as_a_contract_violation() { + for (status, detail) in [ + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_OK, + "success status contained a null batch", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_NULL_CTX_ARG, + "was given a null ctx argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_NULL_OUT_BATCH_ARG, + "was given a null out_batch argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_NULL_OUT_ERROR_ARG, + "was given a null out_error argument", + ), + ] { + assert_eq!( + batch_ext_init_status_to_result(status, ptr::null_mut(), ptr::null_mut()).err(), + Some(LlamaContextLoadError::FfiContract( + crate::FfiContractError { + operation: "llama_rs_batch_ext_init", + detail, + } + )) + ); + } + } + + #[test] + fn initializing_a_batch_reports_memory_exhaustion() { + assert_eq!( + [ + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_ERROR_STRING_ALLOCATION_FAILED, + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_LLAMA_CPP_OUT_OF_MEMORY, + ] + .map(|status| batch_ext_init_status_to_result( + status, + ptr::null_mut(), + ptr::null_mut() + ) + .err()), + [ + Some(LlamaContextLoadError::NotEnoughMemory), + Some(LlamaContextLoadError::LlamaCppOutOfMemory), + ] + ); + } + + #[test] + fn initializing_a_batch_reports_an_exception_message() { + let out_error = unsafe { + llama_cpp_bindings_sys::llama_rs_string_dup(c"batch allocation threw".as_ptr()) + }; + + assert_eq!( + batch_ext_init_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + out_error, + ) + .err(), + Some(LlamaContextLoadError::Reported { + message: "batch allocation threw".to_owned() + }) + ); + } + + #[test] + fn initializing_a_batch_reports_an_exception_without_a_message_as_a_contract_violation() { + assert_eq!( + batch_ext_init_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + ptr::null_mut(), + ) + .err(), + Some(LlamaContextLoadError::FfiContract( + crate::FfiContractError { + operation: "llama_rs_batch_ext_init", + detail: "reported a thrown C++ exception without an error message", + } + )) + ); + } + + #[test] + fn initializing_a_batch_preserves_an_unknown_status() { + assert_eq!( + batch_ext_init_status_to_result(255, ptr::null_mut(), ptr::null_mut()).err(), + Some(LlamaContextLoadError::FfiStatus(crate::FfiStatusError { + operation: "llama_rs_batch_ext_init", + code: 255, + })) + ); + } +} + +#[cfg(test)] +mod batch_ext_add_entry_status_to_result_tests { + use std::ptr; + + use super::batch_ext_add_entry_status_to_result; + use crate::error::batch_transfer_error::BatchTransferError; + + #[test] + fn adding_a_batch_entry_reports_each_refusal_of_a_validated_entry_as_a_contract_violation() { + for (status, detail) in [ + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_BATCH_FULL, + "refused an entry of a batch validated against its limits as full", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_INVALID_TOKEN, + "refused a token validated against the vocabulary", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_INVALID_SEQUENCE_ID, + "refused a sequence id validated against the context", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_RETURNED_UNKNOWN_CODE, + "refused an entry with an undocumented code", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_POSITION_REJECTED, + "refused the position of an entry it had just added", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_OUTPUT_REJECTED, + "refused the output flag of an entry it had just added", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_EMPTY_SEQ_IDS, + "was given an entry without sequence ids", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_BATCH_ARG, + "was given a null batch argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_SEQ_IDS_ARG, + "was given a null seq_ids argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG, + "was given a null out_llama_cpp_return_code argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_OUT_ERROR_ARG, + "was given a null out_error argument", + ), + ] { + assert_eq!( + batch_ext_add_entry_status_to_result(status, ptr::null_mut()), + Err(BatchTransferError::FfiContract(crate::FfiContractError { + operation: "llama_rs_batch_ext_add_entry", + detail, + })) + ); + } + } + + #[test] + fn adding_a_batch_entry_reports_memory_exhaustion() { + assert_eq!( + [ + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_ERROR_STRING_ALLOCATION_FAILED, + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_OUT_OF_MEMORY, + ] + .map(|status| batch_ext_add_entry_status_to_result(status, ptr::null_mut())), + [ + Err(BatchTransferError::NotEnoughMemory), + Err(BatchTransferError::LlamaCppOutOfMemory), + ] + ); + } + + #[test] + fn adding_a_batch_entry_reports_an_exception_message() { + let out_error = unsafe { + llama_cpp_bindings_sys::llama_rs_string_dup(c"entry insertion threw".as_ptr()) + }; + + assert_eq!( + batch_ext_add_entry_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_THREW_CXX_EXCEPTION, + out_error, + ), + Err(BatchTransferError::Reported { + message: "entry insertion threw".to_owned() + }) + ); + } + + #[test] + fn adding_a_batch_entry_reports_an_exception_without_a_message_as_a_contract_violation() { + assert_eq!( + batch_ext_add_entry_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + ), + Err(BatchTransferError::FfiContract(crate::FfiContractError { + operation: "llama_rs_batch_ext_add_entry", + detail: "reported a thrown C++ exception without an error message", + })) + ); + } + + #[test] + fn adding_a_batch_entry_preserves_an_unknown_status() { + assert_eq!( + batch_ext_add_entry_status_to_result(255, ptr::null_mut()), + Err(BatchTransferError::FfiStatus(crate::FfiStatusError { + operation: "llama_rs_batch_ext_add_entry", + code: 255, + })) + ); + } +} + #[cfg(test)] mod decode_status_to_result_tests { use std::ptr; @@ -1421,6 +1671,24 @@ mod decode_status_to_result_tests { assert_eq!(result, Err(DecodeError::NotEnoughMemory)); } + #[test] + fn decode_cxx_exception_reports_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"decode threw".as_ptr()) }; + let result = decode_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_DECODE_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + out_error, + ); + + assert_eq!( + result, + Err(DecodeError::Reported { + message: "decode threw".to_owned() + }) + ); + } + #[test] fn decode_cxx_exception_without_a_message_is_a_contract_error() { let result = decode_status_to_result( @@ -1488,6 +1756,25 @@ mod decode_status_to_result_tests { ); } + #[test] + fn decode_status_to_result_maps_null_batch_arg() { + let outcome = decode_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_DECODE_NULL_BATCH_ARG, + 0, + ptr::null_mut(), + ); + assert_eq!( + outcome.err(), + Some( + crate::FfiContractError { + operation: "llama_rs_decode", + detail: "was given a null batch argument", + } + .into() + ) + ); + } + #[test] fn decode_status_to_result_maps_null_out_error_arg() { let outcome = decode_status_to_result( @@ -1580,6 +1867,24 @@ mod encode_status_to_result_tests { assert_eq!(result, Err(EncodeError::NotEnoughMemory)); } + #[test] + fn encode_cxx_exception_reports_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"encode threw".as_ptr()) }; + let result = encode_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_ENCODE_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + out_error, + ); + + assert_eq!( + result, + Err(EncodeError::Reported { + message: "encode threw".to_owned() + }) + ); + } + #[test] fn encode_cxx_exception_without_a_message_is_a_contract_error() { let result = encode_status_to_result( @@ -1647,6 +1952,25 @@ mod encode_status_to_result_tests { ); } + #[test] + fn encode_status_to_result_maps_null_batch_arg() { + let outcome = encode_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_ENCODE_NULL_BATCH_ARG, + 0, + ptr::null_mut(), + ); + assert_eq!( + outcome.err(), + Some( + crate::FfiContractError { + operation: "llama_rs_encode", + detail: "was given a null batch argument", + } + .into() + ) + ); + } + #[test] fn encode_status_to_result_maps_null_model() { let outcome = encode_status_to_result( diff --git a/llama-cpp-bindings/src/context/kv_cache.rs b/llama-cpp-bindings/src/context/kv_cache.rs index c346b7437..f3cac5a5a 100644 --- a/llama-cpp-bindings/src/context/kv_cache.rs +++ b/llama-cpp-bindings/src/context/kv_cache.rs @@ -397,6 +397,22 @@ mod kv_cache_seq_add_status_to_result_tests { ); } + #[test] + fn add_llama_cpp_exception_status_reports_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"shift threw".as_ptr()) }; + + assert_eq!( + kv_cache_seq_add_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION, + out_error, + ), + Err(KvCacheSeqAddError::Reported { + message: "shift threw".to_owned() + }) + ); + } + #[test] fn add_unknown_status_is_preserved() { let result = kv_cache_seq_add_status_to_result(255, ptr::null_mut()); @@ -410,6 +426,24 @@ mod kv_cache_seq_add_status_to_result_tests { ); } + #[test] + fn kv_cache_seq_add_status_to_result_maps_null_ctx_arg() { + let outcome = kv_cache_seq_add_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_CTX_ARG, + ptr::null_mut(), + ); + assert_eq!( + outcome.err(), + Some( + crate::FfiContractError { + operation: "llama_rs_memory_seq_add", + detail: "was given a null ctx argument", + } + .into() + ) + ); + } + #[test] fn kv_cache_seq_add_status_to_result_maps_null_model() { let outcome = kv_cache_seq_add_status_to_result( @@ -503,6 +537,22 @@ mod kv_cache_seq_div_status_to_result_tests { ); } + #[test] + fn div_llama_cpp_exception_status_reports_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"division threw".as_ptr()) }; + + assert_eq!( + kv_cache_seq_div_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_LLAMA_CPP_THREW_CXX_EXCEPTION, + out_error, + ), + Err(KvCacheSeqDivError::Reported { + message: "division threw".to_owned() + }) + ); + } + #[test] fn div_unknown_status_is_preserved() { let result = kv_cache_seq_div_status_to_result(255, ptr::null_mut()); @@ -516,6 +566,24 @@ mod kv_cache_seq_div_status_to_result_tests { ); } + #[test] + fn kv_cache_seq_div_status_to_result_maps_null_ctx_arg() { + let outcome = kv_cache_seq_div_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_CTX_ARG, + ptr::null_mut(), + ); + assert_eq!( + outcome.err(), + Some( + crate::FfiContractError { + operation: "llama_rs_memory_seq_div", + detail: "was given a null ctx argument", + } + .into() + ) + ); + } + #[test] fn kv_cache_seq_div_status_to_result_maps_null_model() { let outcome = kv_cache_seq_div_status_to_result( @@ -674,6 +742,25 @@ mod kv_cache_seq_pos_max_status_to_result_tests { ); } + #[test] + fn seq_pos_max_llama_cpp_exception_status_reports_its_message() { + let out_error = unsafe { + llama_cpp_bindings_sys::llama_rs_string_dup(c"position lookup threw".as_ptr()) + }; + + assert_eq!( + kv_cache_seq_pos_max_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_LLAMA_CPP_THREW_CXX_EXCEPTION, + -1, + 2, + out_error, + ), + Err(KvCacheSeqPosMaxError::Reported { + message: "position lookup threw".to_owned() + }) + ); + } + #[test] fn seq_pos_max_unknown_status_is_preserved() { assert_eq!( diff --git a/llama-cpp-bindings/src/context/session.rs b/llama-cpp-bindings/src/context/session.rs index 7e30dc05a..5cc36d8ae 100644 --- a/llama-cpp-bindings/src/context/session.rs +++ b/llama-cpp-bindings/src/context/session.rs @@ -1,5 +1,7 @@ use std::ffi::CString; +use std::ffi::c_char; use std::path::Path; +use std::ptr; use crate::context::LlamaContext; use crate::context::llama_state_seq_flags::LlamaStateSeqFlags; @@ -14,7 +16,7 @@ use crate::token::LlamaToken; fn state_data_status_to_result( status: llama_cpp_bindings_sys::llama_rs_state_data_status, byte_count: usize, - out_error: *mut std::ffi::c_char, + out_error: *mut c_char, operation: &'static str, ) -> Result { match status { @@ -70,6 +72,22 @@ fn state_data_status_to_result( } } +fn state_data_restore_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_state_data_status, + restored_bytes: usize, + provided_bytes: usize, + out_error: *mut c_char, + operation: &'static str, +) -> Result { + let restored = state_data_status_to_result(status, restored_bytes, out_error, operation)?; + + if restored == 0 && provided_bytes != 0 { + return Err(StateDataError::NothingRestored { provided_bytes }); + } + + Ok(restored) +} + fn process_session_load_result( success: bool, n_out: usize, @@ -261,7 +279,7 @@ impl LlamaContext<'_> { /// The `dest` buffer must be large enough to hold the complete state data. pub unsafe fn copy_state_data(&self, dest: &mut [u8]) -> Result { let mut byte_count = 0usize; - let mut out_error: *mut std::ffi::c_char = std::ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_state_get_data( self.context.as_ptr(), @@ -286,7 +304,7 @@ impl LlamaContext<'_> { /// Returns [`StateDataError`] when the llama.cpp deserializer rejects the buffer. pub unsafe fn set_state_data(&mut self, src: &[u8]) -> Result { let mut byte_count = 0usize; - let mut out_error: *mut std::ffi::c_char = std::ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_state_set_data( self.context.as_ptr(), @@ -297,16 +315,13 @@ impl LlamaContext<'_> { ) }; - let restored = - state_data_status_to_result(status, byte_count, out_error, "llama_rs_state_set_data")?; - - if restored == 0 && !src.is_empty() { - return Err(StateDataError::NothingRestored { - provided_bytes: src.len(), - }); - } - - Ok(restored) + state_data_restore_status_to_result( + status, + byte_count, + src.len(), + out_error, + "llama_rs_state_set_data", + ) } #[must_use] @@ -334,7 +349,7 @@ impl LlamaContext<'_> { flags: &LlamaStateSeqFlags, ) -> Result { let mut byte_count = 0usize; - let mut out_error: *mut std::ffi::c_char = std::ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_state_seq_get_data( self.context.as_ptr(), @@ -365,7 +380,7 @@ impl LlamaContext<'_> { flags: &LlamaStateSeqFlags, ) -> Result { let mut byte_count = 0usize; - let mut out_error: *mut std::ffi::c_char = std::ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_state_seq_set_data( self.context.as_ptr(), @@ -378,20 +393,13 @@ impl LlamaContext<'_> { ) }; - let restored = state_data_status_to_result( + state_data_restore_status_to_result( status, byte_count, + src.len(), out_error, "llama_rs_state_seq_set_data", - )?; - - if restored == 0 && !src.is_empty() { - return Err(StateDataError::NothingRestored { - provided_bytes: src.len(), - }); - } - - Ok(restored) + ) } } @@ -488,4 +496,135 @@ mod state_data_status_to_result_tests { ); assert_eq!(outcome.err(), Some(StateDataError::LlamaCppOutOfMemory)); } + + #[test] + fn state_data_status_to_result_maps_each_null_argument_to_a_contract_violation_of_the_operation() + { + for (status, detail) in [ + ( + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_CTX_ARG, + "was given a null ctx argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_BUFFER_ARG, + "was given a null buffer argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_OUT_BYTE_COUNT_ARG, + "was given a null out_byte_count argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_OUT_ERROR_ARG, + "was given a null out_error argument", + ), + ] { + assert_eq!( + state_data_status_to_result(status, 0, ptr::null_mut(), "llama_rs_state_get_data"), + Err(StateDataError::FfiContract(crate::FfiContractError { + operation: "llama_rs_state_get_data", + detail, + })) + ); + } + } + + #[test] + fn state_data_status_to_result_maps_error_string_allocation_failed() { + assert_eq!( + state_data_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_ERROR_STRING_ALLOCATION_FAILED, + 0, + ptr::null_mut(), + "llama_rs_state_get_data", + ), + Err(StateDataError::NotEnoughMemory) + ); + } + + #[test] + fn state_data_status_to_result_reports_an_exception_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"serializer threw".as_ptr()) }; + + assert_eq!( + state_data_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + out_error, + "llama_rs_state_get_data", + ), + Err(StateDataError::Reported { + message: "serializer threw".to_owned() + }) + ); + } + + #[test] + fn state_data_status_to_result_maps_an_exception_without_a_message_to_a_contract_violation() { + assert_eq!( + state_data_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + ptr::null_mut(), + "llama_rs_state_get_data", + ), + Err(StateDataError::FfiContract(crate::FfiContractError { + operation: "llama_rs_state_get_data", + detail: "reported a thrown C++ exception without an error message", + })) + ); + } + + #[test] + fn state_data_status_to_result_preserves_an_unknown_status() { + assert_eq!( + state_data_status_to_result(255, 0, ptr::null_mut(), "llama_rs_state_get_data"), + Err(StateDataError::FfiStatus(crate::FfiStatusError { + operation: "llama_rs_state_get_data", + code: 255, + })) + ); + } +} + +#[cfg(test)] +mod state_data_restore_status_to_result_tests { + use std::ptr; + + use super::state_data_restore_status_to_result; + use crate::context::state_data_error::StateDataError; + + #[test] + fn restoring_state_data_fails_only_when_a_non_empty_snapshot_restores_nothing() { + assert_eq!( + [(0, 16), (0, 0), (16, 16)].map(|(restored_bytes, provided_bytes)| { + state_data_restore_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_OK, + restored_bytes, + provided_bytes, + ptr::null_mut(), + "llama_rs_state_seq_set_data", + ) + }), + [ + Err(StateDataError::NothingRestored { provided_bytes: 16 }), + Ok(0), + Ok(16), + ] + ); + } + + #[test] + fn restoring_state_data_propagates_a_failed_status() { + assert_eq!( + state_data_restore_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_LLAMA_CPP_OUT_OF_MEMORY, + 0, + 16, + ptr::null_mut(), + "llama_rs_state_seq_set_data", + ), + Err(StateDataError::LlamaCppOutOfMemory) + ); + } } diff --git a/llama-cpp-bindings/src/error/parse_chat_message_error.rs b/llama-cpp-bindings/src/error/parse_chat_message_error.rs index 073811cc9..cc0fe73be 100644 --- a/llama-cpp-bindings/src/error/parse_chat_message_error.rs +++ b/llama-cpp-bindings/src/error/parse_chat_message_error.rs @@ -10,8 +10,6 @@ pub enum ParseChatMessageError { NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] LlamaCppOutOfMemory, - #[error("the parsed chat destructor threw: {message}")] - DestructorFailed { message: String }, #[error("text llama.cpp parsed out of the answer is not valid UTF-8: {0}")] ParsedTextNotUtf8(#[source] Utf8Error), #[error("{message}")] diff --git a/llama-cpp-bindings/src/grammar_matcher.rs b/llama-cpp-bindings/src/grammar_matcher.rs index 406214372..f7a61e5a5 100644 --- a/llama-cpp-bindings/src/grammar_matcher.rs +++ b/llama-cpp-bindings/src/grammar_matcher.rs @@ -25,6 +25,16 @@ fn panic_payload_message(payload: &(dyn Any + Send)) -> String { "the parser panicked with a payload that carries no message".to_owned() } +fn catch_step_panic( + operation: &'static str, + step: impl FnOnce() -> TValue, +) -> Result { + catch_unwind(AssertUnwindSafe(step)).map_err(|panic_payload| GrammarRuntimeError::Panicked { + operation, + message: panic_payload_message(panic_payload.as_ref()), + }) +} + fn stop_reason_to_result( stop_reason: StopReason, detail: String, @@ -104,9 +114,9 @@ impl GrammarMatcher { operation: &'static str, step: impl FnOnce(&mut TokenParser) -> Result, ) -> Result, GrammarRuntimeError> { - match catch_unwind(AssertUnwindSafe(|| step(&mut self.parser))) { - Ok(Ok(value)) => Ok(StepOutcome::Produced(value)), - Ok(Err(step_error)) => { + match catch_step_panic(operation, || step(&mut self.parser))? { + Ok(value) => Ok(StepOutcome::Produced(value)), + Err(step_error) => { let detail = self .parser .error_message() @@ -115,10 +125,6 @@ impl GrammarMatcher { Ok(StepOutcome::BenignStop) } - Err(panic_payload) => Err(GrammarRuntimeError::Panicked { - operation, - message: panic_payload_message(panic_payload.as_ref()), - }), } } } @@ -127,6 +133,7 @@ impl GrammarMatcher { mod tests { use llguidance::api::StopReason; + use super::catch_step_panic; use super::panic_payload_message; use super::stop_reason_to_result; use crate::error::grammar_runtime_error::GrammarRuntimeError; @@ -155,6 +162,17 @@ mod tests { ); } + #[test] + fn a_panicking_step_is_reported_with_its_operation_and_message() { + assert_eq!( + catch_step_panic("consume_token", || -> u32 { panic!("parser exploded") }), + Err(GrammarRuntimeError::Panicked { + operation: "consume_token", + message: "parser exploded".to_owned(), + }) + ); + } + #[test] fn benign_stop_reasons_are_ok() { for reason in [ diff --git a/llama-cpp-bindings/src/json_schema_to_grammar.rs b/llama-cpp-bindings/src/json_schema_to_grammar.rs index 3eed3495f..2d3434d7c 100644 --- a/llama-cpp-bindings/src/json_schema_to_grammar.rs +++ b/llama-cpp-bindings/src/json_schema_to_grammar.rs @@ -246,6 +246,27 @@ mod json_schema_to_grammar_status_to_result_tests { ); } + #[test] + fn llama_cpp_exception_status_carries_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"conversion threw".as_ptr()) }; + + let result = unsafe { + json_schema_to_grammar_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + out_error, + ) + }; + + assert_eq!( + result, + Err(JsonSchemaToGrammarError::Reported { + message: "conversion threw".to_owned(), + }) + ); + } + #[test] fn allocation_failed_status_returns_not_enough_memory() { let result = unsafe { diff --git a/llama-cpp-bindings/src/lazy_grammar_triggers.rs b/llama-cpp-bindings/src/lazy_grammar_triggers.rs new file mode 100644 index 000000000..74a46ea51 --- /dev/null +++ b/llama-cpp-bindings/src/lazy_grammar_triggers.rs @@ -0,0 +1,258 @@ +use std::ffi::c_char; +use std::ptr; + +use llama_cpp_ffi_status::read_and_free_cpp_string; + +use crate::chat_grammar_trigger::ChatGrammarTrigger; +use crate::error::grammar_error::GrammarError; +use crate::token::LlamaToken; + +fn anchored_pattern(pattern: &str) -> String { + if pattern.is_empty() { + return "^$".to_owned(); + } + + let start = if pattern.starts_with('^') { "" } else { "^" }; + let end = if pattern.ends_with('$') { "" } else { "$" }; + + format!("{start}{pattern}{end}") +} + +fn regex_escape_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_regex_escape_status, + out_string: *mut c_char, + out_error: *mut c_char, +) -> Result { + const OPERATION: &str = "llama_rs_regex_escape"; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_OK => Ok(unsafe { + read_and_free_cpp_string( + out_string, + OPERATION, + "success status contained a null string", + ) + }?), + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_THREW_CXX_EXCEPTION => { + Err(GrammarError::Reported { + message: unsafe { + read_and_free_cpp_string( + out_error, + OPERATION, + "reported a thrown C++ exception without an error message", + ) + }?, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_ERROR_STRING_ALLOCATION_FAILED => { + Err(GrammarError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_OUT_OF_MEMORY => { + Err(GrammarError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_TEXT_ARG => { + Err(crate::FfiContractError { + operation: OPERATION, + detail: "was given a null text argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_OUT_STRING_ARG => { + Err(crate::FfiContractError { + operation: OPERATION, + detail: "was given a null out_string argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_OUT_ERROR_ARG => { + Err(crate::FfiContractError { + operation: OPERATION, + detail: "was given a null out_error argument", + } + .into()) + } + other => Err(crate::FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), + } +} + +fn regex_escaped(word: &str) -> Result { + let mut out_string: *mut c_char = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_regex_escape( + word.as_ptr().cast::(), + word.len(), + &raw mut out_string, + &raw mut out_error, + ) + }; + + regex_escape_status_to_result(status, out_string, out_error) +} + +/// The trigger patterns and tokens that start a lazy grammar, built from a chat session's +/// grammar triggers the way llama.cpp's sampler builds them. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LazyGrammarTriggers { + pub patterns: Vec, + pub tokens: Vec, +} + +impl LazyGrammarTriggers { + /// # Errors + /// + /// Returns the [`GrammarError`] of escaping a trigger word. + pub fn of(triggers: Vec) -> Result { + let mut patterns = Vec::new(); + let mut tokens = Vec::new(); + + for trigger in triggers { + match trigger { + ChatGrammarTrigger::Token(token) => tokens.push(token), + ChatGrammarTrigger::Word(word) => patterns.push(regex_escaped(&word)?), + ChatGrammarTrigger::Pattern(pattern) => patterns.push(pattern), + ChatGrammarTrigger::PatternFull(pattern) => { + patterns.push(anchored_pattern(&pattern)); + } + } + } + + Ok(Self { patterns, tokens }) + } +} + +#[cfg(test)] +mod tests { + use std::ptr; + + use super::LazyGrammarTriggers; + use super::anchored_pattern; + use super::regex_escape_status_to_result; + use crate::chat_grammar_trigger::ChatGrammarTrigger; + use crate::error::grammar_error::GrammarError; + use crate::token::LlamaToken; + + #[test] + fn escaping_a_trigger_word_maps_every_status() { + let escape = |status, out_string, out_error| { + regex_escape_status_to_result(status, out_string, out_error) + }; + let contract = |detail| { + Err(GrammarError::FfiContract(crate::FfiContractError { + operation: "llama_rs_regex_escape", + detail, + })) + }; + + assert_eq!( + [ + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_OK, + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"\\[TOOL\\]".as_ptr()) }, + ptr::null_mut() + ), + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_OK, + ptr::null_mut(), + ptr::null_mut() + ), + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + unsafe { + llama_cpp_bindings_sys::llama_rs_string_dup(c"the escape threw".as_ptr()) + } + ), + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_ERROR_STRING_ALLOCATION_FAILED, + ptr::null_mut(), + ptr::null_mut() + ), + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_OUT_OF_MEMORY, + ptr::null_mut(), + ptr::null_mut() + ), + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_TEXT_ARG, + ptr::null_mut(), + ptr::null_mut() + ), + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_OUT_STRING_ARG, + ptr::null_mut(), + ptr::null_mut() + ), + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_OUT_ERROR_ARG, + ptr::null_mut(), + ptr::null_mut() + ), + escape(255, ptr::null_mut(), ptr::null_mut()), + ], + [ + Ok("\\[TOOL\\]".to_owned()), + contract("success status contained a null string"), + Err(GrammarError::Reported { + message: "the escape threw".to_owned() + }), + Err(GrammarError::NotEnoughMemory), + Err(GrammarError::LlamaCppOutOfMemory), + contract("was given a null text argument"), + contract("was given a null out_string argument"), + contract("was given a null out_error argument"), + Err(GrammarError::FfiStatus(crate::FfiStatusError { + operation: "llama_rs_regex_escape", + code: 255 + })), + ] + ); + } + + #[test] + fn an_escape_failure_without_a_message_is_a_contract_violation() { + assert_eq!( + regex_escape_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + ptr::null_mut() + ), + Err(GrammarError::FfiContract(crate::FfiContractError { + operation: "llama_rs_regex_escape", + detail: "reported a thrown C++ exception without an error message" + })) + ); + } + + #[test] + fn a_full_match_pattern_is_anchored_at_both_ends_once() { + assert_eq!( + ["", "abc", "^abc", "abc$", "^abc$"].map(anchored_pattern), + ["^$", "^abc$", "^abc$", "^abc$", "^abc$"].map(str::to_owned) + ); + } + + #[test] + fn every_trigger_kind_becomes_a_lazy_grammar_trigger() { + assert_eq!( + LazyGrammarTriggers::of(vec![ + ChatGrammarTrigger::Token(LlamaToken::new(151_657)), + ChatGrammarTrigger::Word("[TOOL_CALLS]".to_owned()), + ChatGrammarTrigger::Pattern("".to_owned()), + ChatGrammarTrigger::PatternFull("\\s*\\{".to_owned()), + ]), + Ok(LazyGrammarTriggers { + patterns: vec![ + "\\[TOOL_CALLS\\]".to_owned(), + "".to_owned(), + "^\\s*\\{$".to_owned(), + ], + tokens: vec![LlamaToken::new(151_657)], + }) + ); + } +} diff --git a/llama-cpp-bindings/src/lib.rs b/llama-cpp-bindings/src/lib.rs index 040c10f8d..6ad67b92d 100644 --- a/llama-cpp-bindings/src/lib.rs +++ b/llama-cpp-bindings/src/lib.rs @@ -41,6 +41,7 @@ pub mod ingest_outcome; pub mod invalid_numa_strategy; pub mod json_probe_outcome; pub mod json_schema_to_grammar; +pub mod lazy_grammar_triggers; pub mod llama_backend; pub mod llama_backend_device; pub mod llama_backend_device_type; @@ -68,7 +69,6 @@ pub mod model; pub mod mtmd; pub mod parsed_chat; pub mod raw_chat_message; -pub mod result_after_release; pub mod sampled_token; pub mod sampled_token_classifier; pub mod sampled_token_section; diff --git a/llama-cpp-bindings/src/llguidance_sampler.rs b/llama-cpp-bindings/src/llguidance_sampler.rs index c04457adb..07090280c 100644 --- a/llama-cpp-bindings/src/llguidance_sampler.rs +++ b/llama-cpp-bindings/src/llguidance_sampler.rs @@ -109,15 +109,11 @@ static mut LLG_SAMPLER_I: llama_cpp_bindings_sys::llama_sampler_i = copy_state: None, }; -/// # Errors -/// -/// Returns `GrammarError` if the parser factory, grammar, or parser cannot be created. -pub fn create_llg_sampler( - model: &LlamaModel, +fn create_llg_sampler_from_tok_env( + tok_env: Arc, grammar_kind: &str, grammar_data: &str, ) -> Result { - let tok_env = model.approximate_tok_env()?; let tok_env_dyn: Arc = tok_env.clone(); let factory = llguidance::ParserFactory::new_simple(&tok_env_dyn).map_err(|factory_error| { @@ -157,3 +153,120 @@ pub fn create_llg_sampler( }) .map_err(Into::into) } + +/// # Errors +/// +/// Returns `GrammarError` if the parser factory, grammar, or parser cannot be created. +pub fn create_llg_sampler( + model: &LlamaModel, + grammar_kind: &str, + grammar_data: &str, +) -> Result { + create_llg_sampler_from_tok_env(model.approximate_tok_env()?, grammar_kind, grammar_data) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use anyhow::Result; + use toktrie::ApproximateTokEnv; + use toktrie::INVALID_TOKEN; + use toktrie::TokRxInfo; + use toktrie::TokTrie; + + use super::create_llg_sampler_from_tok_env; + use crate::GrammarError; + use crate::SampleError; + use crate::SamplerAcceptError; + use crate::error::grammar_runtime_error::GrammarRuntimeError; + use crate::sampling::LlamaSampler; + use crate::token::LlamaToken; + use crate::token::data::LlamaTokenData; + use crate::token::data_array::LlamaTokenDataArray; + + const LETTER_A: LlamaToken = LlamaToken::new(0); + const LETTER_B: LlamaToken = LlamaToken::new(1); + const OUT_OF_VOCABULARY: LlamaToken = LlamaToken::new(2); + + fn letter_a_sampler() -> Result { + let vocabulary_without_end_of_generation = TokRxInfo::new(2, INVALID_TOKEN); + let letters = TokTrie::from( + &vocabulary_without_end_of_generation, + &[b"a".to_vec(), b"b".to_vec()], + ); + + create_llg_sampler_from_tok_env(Arc::new(ApproximateTokEnv::new(letters)), "regex", "a") + } + + fn letter_candidates() -> LlamaTokenDataArray { + LlamaTokenDataArray::new( + vec![ + LlamaTokenData::new(LETTER_A, 1.0, 0.0), + LlamaTokenData::new(LETTER_B, 2.0, 0.0), + ], + false, + ) + } + + fn out_of_vocabulary_failure() -> String { + GrammarRuntimeError::InternalParserError { + message: "token id 2 out of range".to_owned(), + } + .to_string() + } + + #[test] + fn apply_leaves_candidates_untouched_once_the_grammar_is_complete() -> Result<()> { + let mut sampler = letter_a_sampler()?; + let mut candidates = letter_candidates(); + sampler.accept(LETTER_A)?; + + sampler.apply(&mut candidates)?; + + assert_eq!(candidates, letter_candidates()); + + Ok(()) + } + + #[test] + fn apply_reports_a_grammar_in_an_error_state_as_a_callback_failure() -> Result<()> { + let mut sampler = letter_a_sampler()?; + + assert_eq!( + sampler.accept(OUT_OF_VOCABULARY), + Err(SamplerAcceptError::GrammarCallbackFailed { + message: out_of_vocabulary_failure(), + }) + ); + assert_eq!( + sampler.apply(&mut letter_candidates()), + Err(SampleError::GrammarCallbackFailed { + message: out_of_vocabulary_failure(), + }) + ); + + Ok(()) + } + + #[test] + fn reset_reports_a_grammar_in_an_error_state_as_a_callback_failure() -> Result<()> { + let mut sampler = letter_a_sampler()?; + sampler.accept(LETTER_A)?; + + assert_eq!( + sampler.accept(OUT_OF_VOCABULARY), + Err(SamplerAcceptError::GrammarCallbackFailed { + message: out_of_vocabulary_failure(), + }) + ); + assert_eq!( + sampler.reset(), + Err(SampleError::GrammarCallbackFailed { + message: out_of_vocabulary_failure(), + }) + ); + + Ok(()) + } +} diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index 81f6a9e2f..9eb05c1c5 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -643,12 +643,11 @@ fn token_piece_bytes_for_tok_env( special: bool, ) -> Result, TokenToStringError> { match model.token_to_piece_bytes(token, TOK_ENV_PIECE_PROBE_SIZE, special, None) { - Ok(bytes) => Ok(bytes), Err(TokenToStringError::UnknownTokenType) => Ok(Vec::new()), Err(TokenToStringError::InsufficientBufferSpace(required)) => { model.token_to_piece_bytes(token, required.unsigned_abs() as usize, special, None) } - Err(other) => Err(other), + probe_result => probe_result, } } @@ -1151,6 +1150,27 @@ mod load_model_from_file_status_to_result_tests { assert_eq!(result.unwrap_err(), LlamaModelLoadError::NotEnoughMemory); } + #[test] + fn load_model_from_file_cxx_exception_reports_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"model loading threw".as_ptr()) }; + let result = unsafe { + load_model_from_file_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + out_error, + Path::new("/some/path"), + ) + }; + + assert_eq!( + result.err(), + Some(LlamaModelLoadError::Reported { + message: "model loading threw".to_owned() + }) + ); + } + #[test] fn load_model_from_file_cxx_exception_without_a_message_is_a_contract_error() { let result = unsafe { @@ -1308,6 +1328,26 @@ mod tokenize_status_to_result_tests { assert_eq!(result, Err(StringToTokenError::NotEnoughMemory)); } + #[test] + fn tokenize_cxx_exception_reports_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"tokenizer threw".as_ptr()) }; + let result = unsafe { + tokenize_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + out_error, + ) + }; + + assert_eq!( + result, + Err(StringToTokenError::Reported { + message: "tokenizer threw".to_owned() + }) + ); + } + #[test] fn tokenize_cxx_exception_without_a_message_is_a_contract_error() { let result = unsafe { diff --git a/llama-cpp-bindings/src/model/params.rs b/llama-cpp-bindings/src/model/params.rs index ce8f46a29..8a9afc72e 100644 --- a/llama-cpp-bindings/src/model/params.rs +++ b/llama-cpp-bindings/src/model/params.rs @@ -772,12 +772,15 @@ mod tests { fn with_devices_too_many_returns_max_exceeded() { let too_many: Vec = (0..17).collect(); let result = LlamaModelParams::default().with_devices(&too_many); + let device_limit = crate::max_devices().min(LLAMA_CPP_MAX_DEVICES); - let Err(LlamaCppError::MaxDevicesExceeded(max_devices)) = result else { - panic!("17 devices must exceed the device limit"); - }; - - assert_eq!(max_devices, crate::max_devices().min(LLAMA_CPP_MAX_DEVICES)); + assert!( + matches!( + result, + Err(LlamaCppError::MaxDevicesExceeded(max_devices)) if max_devices == device_limit + ), + "17 devices must exceed the device limit of {device_limit}, got {result:?}" + ); } #[test] @@ -799,11 +802,13 @@ mod tests { let invalid_index = usize::MAX; let result = LlamaModelParams::default().with_devices(&[invalid_index]); - let Err(LlamaCppError::BackendDeviceNotFound(device)) = result else { - panic!("an out-of-range device index must not be found"); - }; - - assert_eq!(device, invalid_index); + assert!( + matches!( + result, + Err(LlamaCppError::BackendDeviceNotFound(device)) if device == invalid_index + ), + "an out-of-range device index must not be found, got {result:?}" + ); } #[test] @@ -1080,6 +1085,24 @@ mod fit_params_status_to_result_tests { assert_eq!(result, Err(FitError::NotEnoughMemory)); } + #[test] + fn fit_params_status_cxx_exception_reports_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"fitting threw".as_ptr()) }; + let result = fit_params_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + out_error, + ); + + assert_eq!( + result, + Err(FitError::Reported { + message: "fitting threw".to_owned() + }) + ); + } + #[test] fn fit_params_status_cxx_exception_without_a_message_is_a_contract_error_with_unknown_error() { let result = fit_params_status_to_result( diff --git a/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs b/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs index 7895ede72..6486f7fdc 100644 --- a/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs @@ -4,7 +4,6 @@ use std::marker::PhantomData; use crate::context::LlamaContext; use crate::context::decoded_output_rows::DecodedOutputRows; -use crate::token::LlamaToken; use llama_cpp_ffi_status::read_and_free_cpp_string; use super::fitted_mtmd_input_chunks::FittedMtmdInputChunks; @@ -117,13 +116,6 @@ impl<'fitted> FittedMtmdInputChunk<'fitted> { self.chunk_type } - #[must_use] - pub fn text_tokens(&self) -> &[LlamaToken] { - self.chunk - .text_tokens_of(self.chunk_type) - .unwrap_or_default() - } - /// # Errors /// /// Returns any error of [`MtmdEvalLimits::batch_tokens`], diff --git a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs index 98d0d1271..2516cb7bc 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs @@ -429,6 +429,7 @@ mod from_file_status_to_result_tests { use std::ptr; use super::from_file_status_to_result; + use crate::FfiStatusError; use crate::mtmd::mtmd_bitmap_error::MtmdBitmapError; #[test] @@ -582,6 +583,43 @@ mod from_file_status_to_result_tests { }; assert_eq!(outcome.err(), Some(MtmdBitmapError::LlamaCppOutOfMemory)); } + + #[test] + fn from_file_status_llama_cpp_threw_cxx_exception_carries_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"image loader threw".as_ptr()) }; + + let result = unsafe { + from_file_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + out_error, + "/missing/image.png", + ) + }; + + assert_eq!( + result.err(), + Some(MtmdBitmapError::Reported { + message: "image loader threw".to_owned(), + }) + ); + } + + #[test] + fn from_file_unknown_status_is_preserved() { + let result = unsafe { + from_file_status_to_result(255, ptr::null_mut(), ptr::null_mut(), "/missing/image.png") + }; + + assert_eq!( + result.err(), + Some(MtmdBitmapError::FfiStatus(FfiStatusError { + operation: "llama_rs_mtmd_bitmap_init_from_file", + code: 255, + })) + ); + } } #[cfg(test)] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_context.rs b/llama-cpp-bindings/src/mtmd/mtmd_context.rs index 6edd8a45f..5d071405e 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_context.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_context.rs @@ -871,6 +871,23 @@ mod map_tokenize_status_tests { ); assert_eq!(outcome.err(), Some(MtmdTokenizeError::LlamaCppOutOfMemory)); } + + #[test] + fn map_tokenize_status_maps_a_thrown_exception_with_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"tokenizer threw".as_ptr()) }; + + assert_eq!( + map_tokenize_status( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + out_error, + ), + Err(MtmdTokenizeError::Reported { + message: "tokenizer threw".to_owned(), + }) + ); + } } #[cfg(test)] @@ -991,6 +1008,23 @@ mod map_encode_chunk_status_tests { ); assert_eq!(outcome.err(), Some(MtmdEncodeError::LlamaCppOutOfMemory)); } + + #[test] + fn map_encode_chunk_status_maps_a_thrown_exception_with_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"encoder threw".as_ptr()) }; + + assert_eq!( + map_encode_chunk_status( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_ENCODE_CHUNK_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + out_error, + ), + Err(MtmdEncodeError::Reported { + message: "encoder threw".to_owned(), + }) + ); + } } #[cfg(test)] @@ -1133,6 +1167,25 @@ mod map_init_from_file_status_tests { ); assert_eq!(outcome.err(), Some(MtmdInitError::LlamaCppOutOfMemory)); } + + #[test] + fn map_init_from_file_status_maps_a_thrown_exception_with_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"projector threw".as_ptr()) }; + + assert_eq!( + map_init_from_file_status( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + out_error, + "mmproj.gguf", + ) + .err(), + Some(MtmdInitError::Reported { + message: "projector threw".to_owned(), + }) + ); + } } #[cfg(test)] diff --git a/llama-cpp-bindings/src/parsed_chat.rs b/llama-cpp-bindings/src/parsed_chat.rs index 4cf066a75..ff2af3121 100644 --- a/llama-cpp-bindings/src/parsed_chat.rs +++ b/llama-cpp-bindings/src/parsed_chat.rs @@ -1,5 +1,4 @@ use std::ffi::c_char; -use std::ptr; use std::ptr::NonNull; use llama_cpp_bindings_types::ModelProvidedToolCallId; @@ -8,47 +7,9 @@ use llama_cpp_bindings_types::ParsedChatMessage; use llama_cpp_bindings_types::ParsedToolCall; use llama_cpp_bindings_types::ParsedToolCallId; use llama_cpp_bindings_types::ToolCallArguments; -use llama_cpp_ffi_status::read_and_free_cpp_string; use crate::cpp_text_view::cpp_text_view; use crate::error::parse_chat_message_error::ParseChatMessageError; -use crate::result_after_release::result_after_release; - -/// # Safety -/// -/// `free_error` must be the pointer populated by the preceding -/// `llama_rs_parsed_chat_free` call, or null. The destructor-threw arm reads and -/// frees it. -unsafe fn parsed_chat_free_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_parsed_chat_free_status, - free_error: *mut c_char, -) -> Result<(), ParseChatMessageError> { - match status { - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_ERROR_STRING_ALLOCATION_FAILED => { - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - free_error, - "llama_rs_parsed_chat_free", - "reported a thrown C++ exception without an error message", - ) - }?; - - Err(ParseChatMessageError::DestructorFailed { message }) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_free", - code: i64::from(other), - } - .into()), - } -} fn parsed_text( read: impl FnOnce(*mut usize) -> *const c_char, @@ -123,25 +84,19 @@ pub fn take_parsed_chat_message( handle: NonNull, ) -> Result { let message = collect_parsed_chat_message(handle.as_ptr()); - let mut free_error: *mut c_char = ptr::null_mut(); - let free_status = unsafe { - llama_cpp_bindings_sys::llama_rs_parsed_chat_free(handle.as_ptr(), &raw mut free_error) - }; - let freed = unsafe { parsed_chat_free_status_to_result(free_status, free_error) }; - result_after_release(message, freed) + unsafe { llama_cpp_bindings_sys::llama_rs_parsed_chat_free(handle.as_ptr()) }; + + message } #[cfg(test)] mod tests { - use std::ptr; use llama_cpp_bindings_types::ModelProvidedToolCallId; use llama_cpp_bindings_types::ParsedToolCallId; - use super::parsed_chat_free_status_to_result; use super::tool_call_id_reported_by_llama_cpp; - use crate::error::parse_chat_message_error::ParseChatMessageError; #[test] fn an_empty_llama_cpp_tool_call_id_is_not_provided_by_the_model() { @@ -159,79 +114,4 @@ mod tests { .map(ParsedToolCallId::ModelProvided) ); } - - #[test] - fn parsed_chat_free_ok_is_success() { - let result = unsafe { - parsed_chat_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_OK, - ptr::null_mut(), - ) - }; - - assert!( - result.is_ok(), - "a clean destructor must not report a failure" - ); - } - - #[test] - fn parsed_chat_free_allocation_failed_is_not_enough_memory() { - let result = unsafe { - parsed_chat_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - ) - }; - - assert_eq!(result.err(), Some(ParseChatMessageError::NotEnoughMemory)); - } - - #[test] - fn parsed_chat_free_llama_cpp_out_of_memory_is_preserved() { - let result = unsafe { - parsed_chat_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null_mut(), - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::LlamaCppOutOfMemory) - ); - } - - #[test] - fn parsed_chat_free_destructor_threw_surfaces_the_message() { - let out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"the destructor threw".as_ptr()) - }; - let result = unsafe { - parsed_chat_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - out_error, - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::DestructorFailed { - message: "the destructor threw".to_owned() - }) - ); - } - - #[test] - fn parsed_chat_free_unknown_status_is_preserved() { - let result = unsafe { parsed_chat_free_status_to_result(255, ptr::null_mut()) }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_free", - code: 255, - })) - ); - } } diff --git a/llama-cpp-bindings/src/result_after_release.rs b/llama-cpp-bindings/src/result_after_release.rs deleted file mode 100644 index b99b8b72d..000000000 --- a/llama-cpp-bindings/src/result_after_release.rs +++ /dev/null @@ -1,50 +0,0 @@ -use std::fmt::Display; - -/// Combines the result of reading a llama.cpp handle with the result of releasing it: a read -/// failure wins and a release failure that accompanies it is logged, since only one error can be -/// returned. -/// -/// # Errors -/// -/// Returns the read failure, or the release failure after a successful read. -pub fn result_after_release( - read: Result, - released: Result<(), TError>, -) -> Result { - match read { - Ok(value) => released.map(|()| value), - Err(read_failure) => { - if let Err(release_failure) = released { - log::error!("{release_failure}"); - } - - Err(read_failure) - } - } -} - -#[cfg(test)] -mod tests { - use super::result_after_release; - - #[test] - fn a_successful_read_and_release_yields_the_value() { - assert_eq!(result_after_release::(Ok(7), Ok(())), Ok(7)); - } - - #[test] - fn a_failed_release_after_a_successful_read_is_returned() { - assert_eq!( - result_after_release::(Ok(7), Err("release".to_owned())), - Err("release".to_owned()) - ); - } - - #[test] - fn a_failed_read_wins_over_a_failed_release() { - assert_eq!( - result_after_release::(Err("read".to_owned()), Err("release".to_owned())), - Err("read".to_owned()) - ); - } -} diff --git a/llama-cpp-bindings/src/sampled_token.rs b/llama-cpp-bindings/src/sampled_token.rs index 67b91cf70..94f13957c 100644 --- a/llama-cpp-bindings/src/sampled_token.rs +++ b/llama-cpp-bindings/src/sampled_token.rs @@ -1,3 +1,4 @@ +use crate::sampled_token_section::SampledTokenSection; use crate::token::LlamaToken; #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] @@ -9,6 +10,26 @@ pub enum SampledToken { } impl SampledToken { + #[must_use] + pub const fn in_section(section: SampledTokenSection, token: LlamaToken) -> Self { + match section { + SampledTokenSection::Reasoning => Self::Reasoning(token), + SampledTokenSection::Content => Self::Content(token), + SampledTokenSection::ToolCall => Self::ToolCall(token), + SampledTokenSection::Pending => Self::Undeterminable(token), + } + } + + #[must_use] + pub const fn section(&self) -> SampledTokenSection { + match self { + Self::Reasoning(_) => SampledTokenSection::Reasoning, + Self::Content(_) => SampledTokenSection::Content, + Self::ToolCall(_) => SampledTokenSection::ToolCall, + Self::Undeterminable(_) => SampledTokenSection::Pending, + } + } + #[must_use] pub const fn token(&self) -> LlamaToken { match *self { @@ -23,6 +44,7 @@ impl SampledToken { #[cfg(test)] mod tests { use super::SampledToken; + use crate::sampled_token_section::SampledTokenSection; use crate::token::LlamaToken; #[test] @@ -40,4 +62,28 @@ mod tests { [token; 4] ); } + + #[test] + fn a_token_placed_in_a_section_reports_that_section() { + let token = LlamaToken::new(7); + let sections = [ + SampledTokenSection::Pending, + SampledTokenSection::Content, + SampledTokenSection::Reasoning, + SampledTokenSection::ToolCall, + ]; + + assert_eq!( + sections.map(|section| SampledToken::in_section(section, token).section()), + sections + ); + } + + #[test] + fn a_token_of_no_section_is_undeterminable() { + assert_eq!( + SampledToken::in_section(SampledTokenSection::Pending, LlamaToken::new(7)), + SampledToken::Undeterminable(LlamaToken::new(7)) + ); + } } diff --git a/llama-cpp-bindings/src/sampled_token_classifier.rs b/llama-cpp-bindings/src/sampled_token_classifier.rs index 05f58720a..f47b2460d 100644 --- a/llama-cpp-bindings/src/sampled_token_classifier.rs +++ b/llama-cpp-bindings/src/sampled_token_classifier.rs @@ -7,7 +7,6 @@ use llama_cpp_bindings_types::TokenUsageError; use crate::bare_json_tool_call_probe::BareJsonToolCallProbe; use crate::bare_json_tool_calls::BareJsonToolCalls; use crate::batch_add_error::BatchAddError; -use crate::chat_classification::ChatClassification; use crate::chat_output::ChatOutput; use crate::chat_output_completeness::ChatOutputCompleteness; use crate::chat_piece_role::ChatPieceRole; @@ -69,27 +68,9 @@ enum TokenAttribution<'session, 'model> { ChatSession(ChatSessionAttribution<'session, 'model>), } -const fn sampled_token_in(section: SampledTokenSection, token: LlamaToken) -> SampledToken { - match section { - SampledTokenSection::Reasoning => SampledToken::Reasoning(token), - SampledTokenSection::Content => SampledToken::Content(token), - SampledTokenSection::ToolCall => SampledToken::ToolCall(token), - SampledTokenSection::Pending => SampledToken::Undeterminable(token), - } -} - -const fn section_of(sampled_token: SampledToken) -> SampledTokenSection { - match sampled_token { - SampledToken::Reasoning(_) => SampledTokenSection::Reasoning, - SampledToken::Content(_) => SampledTokenSection::Content, - SampledToken::ToolCall(_) => SampledTokenSection::ToolCall, - SampledToken::Undeterminable(_) => SampledTokenSection::Pending, - } -} - const fn outcome_of(token: LlamaToken, role: ChatPieceRole, piece: String) -> IngestOutcome { IngestOutcome { - sampled_token: sampled_token_in(role.section(), token), + sampled_token: SampledToken::in_section(role.section(), token), piece: match role { ChatPieceRole::Text(_) => TokenPiece::Visible(piece), ChatPieceRole::Marker(_) => TokenPiece::Marker(piece), @@ -226,17 +207,7 @@ impl<'session, 'model> SampledTokenClassifier<'session, 'model> { .tokens() .get(attribution.attributed_tokens) { - let role = match (&classification, completeness) { - (ChatClassification::Spans(spans), _) => spans.classify_token(&output_token.bytes), - (ChatClassification::OutputDoesNotMatchFormat, ChatOutputCompleteness::Partial) => { - None - } - ( - ChatClassification::OutputDoesNotMatchFormat, - ChatOutputCompleteness::Finished, - ) => Some(ChatPieceRole::Text(SampledTokenSection::Pending)), - }; - let Some(role) = role else { + let Some(role) = classification.classify_token(&output_token.bytes) else { break; }; let piece = attribution.output.text()[output_token.bytes.clone()].to_owned(); @@ -252,7 +223,7 @@ impl<'session, 'model> SampledTokenClassifier<'session, 'model> { } for outcome in released { - self.section = section_of(outcome.sampled_token); + self.section = outcome.sampled_token.section(); self.record_usage_in(self.section); outcomes.push(outcome); } diff --git a/llama-cpp-bindings/src/sampling.rs b/llama-cpp-bindings/src/sampling.rs index 1a9ca147e..8f44e019a 100644 --- a/llama-cpp-bindings/src/sampling.rs +++ b/llama-cpp-bindings/src/sampling.rs @@ -1288,6 +1288,8 @@ mod check_sampler_accept_status_tests { use std::ptr; use super::check_sampler_accept_status; + use crate::FfiContractError; + use crate::FfiStatusError; use crate::SamplerAcceptError; #[test] @@ -1334,7 +1336,7 @@ mod check_sampler_accept_status_tests { assert_eq!( result, - Err(SamplerAcceptError::FfiContract(crate::FfiContractError { + Err(SamplerAcceptError::FfiContract(FfiContractError { operation: "llama_rs_sampler_accept", detail: "was given a null sampler argument", })) @@ -1350,7 +1352,7 @@ mod check_sampler_accept_status_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_accept", detail: "was given a null out_error argument", } @@ -1367,6 +1369,37 @@ mod check_sampler_accept_status_tests { ); assert_eq!(outcome.err(), Some(SamplerAcceptError::LlamaCppOutOfMemory)); } + + #[test] + fn check_sampler_accept_status_exception_without_a_message_is_a_contract_error() { + assert_eq!( + check_sampler_accept_status( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + ), + Err(SamplerAcceptError::FfiContract(FfiContractError { + operation: "llama_rs_sampler_accept", + detail: "reported a thrown C++ exception without an error message", + })) + ); + } + + #[test] + fn check_sampler_accept_status_maps_an_unknown_status() { + assert_eq!( + check_sampler_accept_status( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_LLAMA_CPP_THREW_CXX_EXCEPTION + 1, + ptr::null_mut(), + ), + Err(SamplerAcceptError::FfiStatus(FfiStatusError { + operation: "llama_rs_sampler_accept", + code: i64::from( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_LLAMA_CPP_THREW_CXX_EXCEPTION + + 1 + ), + })) + ); + } } #[cfg(test)] @@ -1374,6 +1407,7 @@ mod sampler_sample_status_to_result_tests { use std::ptr; use super::sampler_sample_status_to_result; + use crate::FfiStatusError; use crate::SampleError; #[test] @@ -1485,6 +1519,41 @@ mod sampler_sample_status_to_result_tests { ); assert_eq!(outcome.err(), Some(SampleError::LlamaCppOutOfMemory)); } + + #[test] + fn sampler_sample_status_exception_carries_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"sampler threw".as_ptr()) }; + + assert_eq!( + sampler_sample_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_LLAMA_CPP_THREW_CXX_EXCEPTION, + -1, + out_error, + ), + Err(SampleError::Reported { + message: "sampler threw".to_owned(), + }) + ); + } + + #[test] + fn sampler_sample_status_to_result_maps_an_unknown_status() { + assert_eq!( + sampler_sample_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_LLAMA_CPP_THREW_CXX_EXCEPTION + 1, + -1, + ptr::null_mut(), + ), + Err(SampleError::FfiStatus(FfiStatusError { + operation: "llama_rs_sampler_sample", + code: i64::from( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_LLAMA_CPP_THREW_CXX_EXCEPTION + + 1 + ), + })) + ); + } } #[cfg(test)] @@ -1773,6 +1842,7 @@ mod sampler_init_grammar_status_to_result_tests { use std::ptr; use super::sampler_init_grammar_status_to_result; + use crate::FfiStatusError; use crate::GrammarError; #[test] @@ -1858,6 +1928,44 @@ mod sampler_init_grammar_status_to_result_tests { ); assert_eq!(outcome.err(), Some(GrammarError::LlamaCppOutOfMemory)); } + + #[test] + fn sampler_init_grammar_status_exception_carries_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"grammar threw".as_ptr()) }; + + assert_eq!( + sampler_init_grammar_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + out_error, + ) + .err(), + Some(GrammarError::Reported { + message: "grammar threw".to_owned(), + }) + ); + } + + #[test] + fn sampler_init_grammar_status_to_result_maps_an_unknown_status() { + assert_eq!( + sampler_init_grammar_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LLAMA_CPP_THREW_CXX_EXCEPTION + + 1, + ptr::null_mut(), + ptr::null_mut(), + ) + .err(), + Some(GrammarError::FfiStatus(FfiStatusError { + operation: "llama_rs_sampler_init_grammar", + code: i64::from( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LLAMA_CPP_THREW_CXX_EXCEPTION + + 1 + ), + })) + ); + } } #[cfg(test)] @@ -1865,6 +1973,7 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { use std::ptr; use super::sampler_init_grammar_lazy_patterns_status_to_result; + use crate::FfiStatusError; use crate::GrammarError; #[test] @@ -1949,4 +2058,42 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { let outcome = sampler_init_grammar_lazy_patterns_status_to_result(llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), ptr::null_mut()); assert_eq!(outcome.err(), Some(GrammarError::LlamaCppOutOfMemory)); } + + #[test] + fn sampler_init_grammar_lazy_patterns_status_exception_carries_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"lazy grammar threw".as_ptr()) }; + + assert_eq!( + sampler_init_grammar_lazy_patterns_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + out_error, + ) + .err(), + Some(GrammarError::Reported { + message: "lazy grammar threw".to_owned(), + }) + ); + } + + #[test] + fn sampler_init_grammar_lazy_patterns_status_to_result_maps_an_unknown_status() { + let unknown_status = + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_LLAMA_CPP_THREW_CXX_EXCEPTION + + 1; + + assert_eq!( + sampler_init_grammar_lazy_patterns_status_to_result( + unknown_status, + ptr::null_mut(), + ptr::null_mut(), + ) + .err(), + Some(GrammarError::FfiStatus(FfiStatusError { + operation: "llama_rs_sampler_init_grammar_lazy_patterns", + code: i64::from(unknown_status), + })) + ); + } } diff --git a/llama-cpp-bindings/src/streaming_json_probe.rs b/llama-cpp-bindings/src/streaming_json_probe.rs index 816398f8c..d48241b87 100644 --- a/llama-cpp-bindings/src/streaming_json_probe.rs +++ b/llama-cpp-bindings/src/streaming_json_probe.rs @@ -596,4 +596,30 @@ mod tests { JsonProbeOutcome::Failed, ); } + + #[test] + fn whitespace_around_colons_and_after_values_is_completed_valid() { + assert_eq!( + probe(r#"{"name" : "f" , "arguments" : {} }"#), + JsonProbeOutcome::CompletedValid, + ); + } + + #[test] + fn key_not_followed_by_a_colon_is_failed() { + assert_eq!(probe(r#"{"name""f"}"#), JsonProbeOutcome::Failed); + } + + #[test] + fn value_followed_by_neither_a_comma_nor_a_close_is_failed() { + assert_eq!( + probe(r#"{"name":"f""arguments":{}}"#), + JsonProbeOutcome::Failed + ); + } + + #[test] + fn trailing_comma_before_the_close_is_failed() { + assert_eq!(probe(r#"{"name":"f",}"#), JsonProbeOutcome::Failed); + } } diff --git a/llama-cpp-bindings/src/token/data_array.rs b/llama-cpp-bindings/src/token/data_array.rs index 497121898..1fb73370e 100644 --- a/llama-cpp-bindings/src/token/data_array.rs +++ b/llama-cpp-bindings/src/token/data_array.rs @@ -583,4 +583,20 @@ mod sampler_apply_status_to_result_tests { ); assert_eq!(outcome.err(), Some(SamplerApplyError::LlamaCppOutOfMemory)); } + + #[test] + fn sampler_apply_status_cxx_exception_carries_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"apply threw".as_ptr()) }; + + assert_eq!( + sampler_apply_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_THREW_CXX_EXCEPTION, + out_error, + ), + Err(SamplerApplyError::Reported { + message: "apply threw".to_owned(), + }) + ); + } } diff --git a/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs b/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs index 0e1648563..66801eb7b 100644 --- a/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs +++ b/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs @@ -208,6 +208,7 @@ pub fn parse( mod tests { use llama_cpp_bindings_types::PairedQuoteShape; use llama_cpp_bindings_types::ParsedToolCall; + use llama_cpp_bindings_types::ParsedToolCallId; use llama_cpp_bindings_types::ToolCallArgsShape; use llama_cpp_bindings_types::ToolCallArguments; use llama_cpp_bindings_types::ToolCallMarkers; @@ -305,6 +306,24 @@ mod tests { ); } + #[test] + fn parses_comma_separated_bare_values() { + let parsed = parse( + &gemma4_call("a:42,b:true"), + &gemma4_markers(), + &gemma4_shape(), + ); + + assert_eq!( + parsed, + Ok(vec![ParsedToolCall::new( + ParsedToolCallId::NotProvidedByModel, + "f".to_owned(), + ToolCallArguments::ValidJson(json!({"a": 42, "b": true})), + )]), + ); + } + #[test] fn parses_bare_boolean_value() { let parsed = diff --git a/llama-cpp-test-harness-macros/src/expand.rs b/llama-cpp-test-harness-macros/src/expand.rs index 1e27074e0..52e99c909 100644 --- a/llama-cpp-test-harness-macros/src/expand.rs +++ b/llama-cpp-test-harness-macros/src/expand.rs @@ -433,13 +433,16 @@ mod tests { assert_eq!( items .iter() - .map(|item| match item { - Item::Fn(_) => "fn", - Item::Macro(_) => "registration", - _ => "other", - }) - .collect::>(), - ["fn", "registration", "registration"] + .map(|item| matches!(item, Item::Fn(_))) + .collect::>(), + [true, false, false] + ); + assert_eq!( + items + .iter() + .map(|item| matches!(item, Item::Macro(_))) + .collect::>(), + [false, true, true] ); } } From a62e7859fafe3b9f960fc549fcc9402a10000a44 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Fri, 30 Oct 2026 13:04:38 +0100 Subject: [PATCH 05/82] Release chat handles through temporary owners so cppcheck sees no unread variables --- llama-cpp-bindings-sys/wrapper_chat_session.cpp | 4 ++-- llama-cpp-bindings-sys/wrapper_chat_templates.cpp | 2 +- llama-cpp-bindings-sys/wrapper_parsed_chat.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/llama-cpp-bindings-sys/wrapper_chat_session.cpp b/llama-cpp-bindings-sys/wrapper_chat_session.cpp index 728b6a41b..4efee6f96 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_session.cpp +++ b/llama-cpp-bindings-sys/wrapper_chat_session.cpp @@ -242,7 +242,7 @@ extern "C" auto llama_rs_chat_session_create( } extern "C" auto llama_rs_chat_session_free(llama_rs_chat_session_handle session) -> void { - const std::unique_ptr reclaimed(session); + std::unique_ptr(session).reset(); } extern "C" auto llama_rs_chat_session_prompt_part_count(llama_rs_chat_session_handle session) -> size_t { @@ -498,7 +498,7 @@ extern "C" auto llama_rs_chat_spans_end(llama_rs_chat_spans_handle spans, size_t } extern "C" auto llama_rs_chat_spans_free(llama_rs_chat_spans_handle spans) -> void { - const std::unique_ptr reclaimed(spans); + std::unique_ptr(spans).reset(); } extern "C" auto llama_rs_regex_escape( diff --git a/llama-cpp-bindings-sys/wrapper_chat_templates.cpp b/llama-cpp-bindings-sys/wrapper_chat_templates.cpp index 45d3a6dc3..dddd021e1 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_templates.cpp +++ b/llama-cpp-bindings-sys/wrapper_chat_templates.cpp @@ -65,7 +65,7 @@ extern "C" auto llama_rs_chat_templates_init( } extern "C" auto llama_rs_chat_templates_free(llama_rs_chat_templates_handle templates) -> void { - const std::unique_ptr reclaimed(templates); + std::unique_ptr(templates).reset(); } extern "C" auto llama_rs_chat_templates_supports_tool_calls(llama_rs_chat_templates_handle templates) -> bool { diff --git a/llama-cpp-bindings-sys/wrapper_parsed_chat.cpp b/llama-cpp-bindings-sys/wrapper_parsed_chat.cpp index f9f32c99a..b0e87be3b 100644 --- a/llama-cpp-bindings-sys/wrapper_parsed_chat.cpp +++ b/llama-cpp-bindings-sys/wrapper_parsed_chat.cpp @@ -6,7 +6,7 @@ #include extern "C" auto llama_rs_parsed_chat_free(llama_rs_parsed_chat_handle handle) -> void { - const std::unique_ptr reclaimed(handle); + std::unique_ptr(handle).reset(); } extern "C" auto llama_rs_parsed_chat_content(llama_rs_parsed_chat_handle handle, size_t * out_length) From 291d790e9450aabf4c67e1e8b81e6cdbde323308 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Fri, 30 Oct 2026 13:09:07 +0100 Subject: [PATCH 06/82] Assert exact strings in tests, name the sampler correctly in Debug, and parse target-cpu from codegen flags as rustc does --- llama-cpp-bindings-build/src/cmake_config.rs | 85 +++++++++++- .../src/model/llama_chat_template.rs | 2 +- llama-cpp-bindings/src/model/params.rs | 28 ++-- llama-cpp-bindings/src/sampling.rs | 4 +- llama-cpp-bindings/src/send_logs_to_log.rs | 103 +++++++++----- .../src/read_and_free_cpp_string.rs | 8 +- llama-cpp-test-harness-macros/src/lib.rs | 22 ++- .../src/parsed_args.rs | 131 ++++++++++-------- .../src/parsed_load_mode.rs | 7 +- .../src/parsed_source.rs | 48 +++++-- 10 files changed, 298 insertions(+), 140 deletions(-) diff --git a/llama-cpp-bindings-build/src/cmake_config.rs b/llama-cpp-bindings-build/src/cmake_config.rs index 768d78a26..f688bde40 100644 --- a/llama-cpp-bindings-build/src/cmake_config.rs +++ b/llama-cpp-bindings-build/src/cmake_config.rs @@ -79,18 +79,39 @@ fn configure_base_defines(config: &mut Config) { config.define("LLAMA_OPENSSL", "OFF"); } +fn codegen_options(encoded_rustflags: &str) -> Vec<&str> { + let mut options = Vec::new(); + let mut flags = encoded_rustflags.split('\x1f'); + + while let Some(flag) = flags.next() { + if flag == "-C" || flag == "--codegen" { + options.extend(flags.next()); + } else if let Some(option) = flag + .strip_prefix("--codegen=") + .or_else(|| flag.strip_prefix("-C")) + { + options.push(option); + } + } + + options +} + +fn target_cpu_of(encoded_rustflags: &str) -> Option { + codegen_options(encoded_rustflags) + .into_iter() + .rev() + .find_map(|option| option.strip_prefix("target-cpu=")) + .map(ToOwned::to_owned) +} + fn configure_cpu_features( config: &mut Config, cargo_cfg_target_arch: &str, target_os: TargetOs, ) -> Result<(), BuildError> { - let target_cpu = optional_env("CARGO_ENCODED_RUSTFLAGS")?.and_then(|rustflags| { - rustflags - .split('\x1f') - .find(|flag| flag.contains("target-cpu=")) - .and_then(|flag| flag.split("target-cpu=").nth(1)) - .map(ToString::to_string) - }); + let target_cpu = + optional_env("CARGO_ENCODED_RUSTFLAGS")?.and_then(|rustflags| target_cpu_of(&rustflags)); if target_cpu.as_deref() == Some("native") { debug_log!("Detected target-cpu=native, compiling with GGML_NATIVE"); @@ -295,3 +316,53 @@ mod cpu_feature_mapping_tests { assert_eq!(map_cpu_feature_to_ggml(""), None); } } + +#[cfg(test)] +mod target_cpu_of_tests { + use super::target_cpu_of; + + #[test] + fn an_attached_codegen_target_cpu_is_found() { + assert_eq!( + target_cpu_of("-Ctarget-cpu=native"), + Some("native".to_owned()) + ); + } + + #[test] + fn a_codegen_target_cpu_given_as_the_next_flag_is_found() { + assert_eq!( + [ + target_cpu_of("-C\x1ftarget-cpu=znver4"), + target_cpu_of("--codegen\x1ftarget-cpu=znver4"), + target_cpu_of("--codegen=target-cpu=znver4"), + ], + [ + Some("znver4".to_owned()), + Some("znver4".to_owned()), + Some("znver4".to_owned()), + ] + ); + } + + #[test] + fn the_last_target_cpu_wins_as_it_does_for_rustc() { + assert_eq!( + target_cpu_of("-Ctarget-cpu=haswell\x1f-Ctarget-cpu=znver4"), + Some("znver4".to_owned()) + ); + } + + #[test] + fn a_target_cpu_inside_another_codegen_option_is_not_the_target_cpu() { + assert_eq!(target_cpu_of("-Clink-arg=-Wl,target-cpu=znver4"), None); + } + + #[test] + fn rustflags_without_a_target_cpu_name_no_target_cpu() { + assert_eq!( + target_cpu_of("-Copt-level=3\x1f--cfg\x1ffeature=\"x\""), + None + ); + } +} diff --git a/llama-cpp-bindings/src/model/llama_chat_template.rs b/llama-cpp-bindings/src/model/llama_chat_template.rs index 3e8f86d02..b752b7bd5 100644 --- a/llama-cpp-bindings/src/model/llama_chat_template.rs +++ b/llama-cpp-bindings/src/model/llama_chat_template.rs @@ -59,7 +59,7 @@ mod tests { let template = LlamaChatTemplate::new("chatml").unwrap(); let debug_output = format!("{template:?}"); - assert!(debug_output.contains("chatml")); + assert_eq!(debug_output, "\"chatml\""); } #[test] diff --git a/llama-cpp-bindings/src/model/params.rs b/llama-cpp-bindings/src/model/params.rs index 8a9afc72e..c651e3fe7 100644 --- a/llama-cpp-bindings/src/model/params.rs +++ b/llama-cpp-bindings/src/model/params.rs @@ -670,24 +670,36 @@ mod tests { assert!(!params.no_alloc()); } + fn debug_output_of_default_params_with(lazy_mode: LlamaLazyMode) -> String { + format!( + "LlamaModelParams {{ n_gpu_layers: -1, main_gpu: 0, vocab_only: false, \ + lazy_mode: Ok({lazy_mode:?}), load_mode: Ok(Auto), load_mtp: false, \ + split_mode: Ok(Layer), devices: [0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, \ + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0], \ + kv_overrides: \"vec of kv_overrides\", .. }}" + ) + } + #[test] fn debug_format_contains_field_names() { let params = LlamaModelParams::default(); let debug_output = format!("{params:?}"); - assert!(debug_output.contains("n_gpu_layers")); - assert!(debug_output.contains("main_gpu")); - assert!(debug_output.contains("vocab_only")); - assert!(debug_output.contains("load_mode")); - assert!(debug_output.contains("split_mode")); + assert_eq!( + debug_output, + debug_output_of_default_params_with(LlamaLazyMode::Auto) + ); } #[test] fn debug_format_includes_lazy_mode() { - let debug_output = format!("{:?}", LlamaModelParams::default()); + let params = LlamaModelParams::default().with_lazy_mode(LlamaLazyMode::On); + let debug_output = format!("{params:?}"); - assert!(debug_output.contains("lazy_mode")); - assert!(debug_output.contains("Auto")); + assert_eq!( + debug_output, + debug_output_of_default_params_with(LlamaLazyMode::On) + ); } #[test] diff --git a/llama-cpp-bindings/src/sampling.rs b/llama-cpp-bindings/src/sampling.rs index 8f44e019a..7d1a4b27c 100644 --- a/llama-cpp-bindings/src/sampling.rs +++ b/llama-cpp-bindings/src/sampling.rs @@ -357,7 +357,7 @@ fn grammar_callback_error_to_accept_result( impl Debug for LlamaSampler { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("LlamaSamplerChain").finish() + f.debug_struct("LlamaSampler").finish_non_exhaustive() } } @@ -1215,7 +1215,7 @@ mod tests { fn debug_formatting() { let sampler = greedy_sampler(); let debug_output = format!("{sampler:?}"); - assert!(debug_output.contains("LlamaSampler")); + assert_eq!(debug_output, "LlamaSampler { .. }"); } } diff --git a/llama-cpp-bindings/src/send_logs_to_log.rs b/llama-cpp-bindings/src/send_logs_to_log.rs index 9c5fa7e19..93f1a005b 100644 --- a/llama-cpp-bindings/src/send_logs_to_log.rs +++ b/llama-cpp-bindings/src/send_logs_to_log.rs @@ -180,7 +180,7 @@ mod tests { }; use crate::log_options::LogOptions; - #[derive(Clone, Debug)] + #[derive(Debug)] struct CapturedRecord { level: Level, target: String, @@ -224,7 +224,7 @@ mod tests { }); } - fn records_for(target: &str) -> Vec { + fn records_for(target: &str) -> Vec { let guard = TEST_LOGGER .records .lock() @@ -232,7 +232,10 @@ mod tests { guard .iter() .filter(|record| record.target == target) - .cloned() + .map(|record| LogRecord { + level: record.level, + text: record.message.clone(), + }) .collect() } @@ -308,7 +311,7 @@ mod tests { &source, ); - assert!(records_for(target).is_empty()); + assert_eq!(records_for(target), Vec::::new()); } #[test] @@ -326,9 +329,13 @@ mod tests { &source, ); - assert!(records_for(target).iter().any(|record| { - record.level == Level::Debug && record.message.contains("info-line") - })); + assert_eq!( + records_for(target), + [LogRecord { + level: Level::Debug, + text: "info-line".to_owned(), + }] + ); } #[test] @@ -346,10 +353,12 @@ mod tests { &source, ); - assert!( - records_for(target).iter().any(|record| { - record.level == Level::Warn && record.message.contains("warn-line") - }) + assert_eq!( + records_for(target), + [LogRecord { + level: Level::Warn, + text: "warn-line".to_owned(), + }] ); } @@ -361,11 +370,13 @@ mod tests { let source = LogSource::new(target, LogOptions::default()); invoke_callback(9999, c"weird\n", &source); - assert!(records_for(target).iter().any(|record| { - record.level == Level::Warn - && record.message.contains("[unknown level 9999]") - && record.message.contains("weird") - })); + assert_eq!( + records_for(target), + [LogRecord { + level: Level::Warn, + text: "[unknown level 9999] weird".to_owned(), + }] + ); } #[test] @@ -380,9 +391,19 @@ mod tests { &source, ); - assert!(records_for(target).iter().any(|record| { - record.level == Level::Warn && record.message.contains("OrphanCont") - })); + assert_eq!( + records_for(target), + [ + LogRecord { + level: Level::Info, + text: "ghost".to_owned(), + }, + LogRecord { + level: Level::Warn, + text: "llama.cpp log decoder anomaly: OrphanCont".to_owned(), + }, + ] + ); } #[test] @@ -410,7 +431,7 @@ mod tests { let source = LogSource::new(target, LogOptions::default()); dispatch_output(&source, DecodeOutput::None); - assert!(records_for(target).is_empty()); + assert_eq!(records_for(target), Vec::::new()); } #[test] @@ -433,16 +454,18 @@ mod tests { }, ); - let records = records_for(target); - assert!( - records - .iter() - .any(|record| record.message.contains("earlier-line")) - ); - assert!( - records - .iter() - .any(|record| record.message.contains("current-line")) + assert_eq!( + records_for(target), + [ + LogRecord { + level: Level::Info, + text: "earlier-line".to_owned(), + }, + LogRecord { + level: Level::Warn, + text: "current-line".to_owned(), + }, + ] ); } @@ -473,9 +496,13 @@ mod tests { ); } - assert!(records_for(target).iter().any(|record| { - record.level == Level::Warn && record.message.contains("NULL text pointer") - })); + assert_eq!( + records_for(target), + [LogRecord { + level: Level::Warn, + text: "received NULL text pointer from llama.cpp log callback".to_owned(), + }] + ); } #[test] @@ -504,10 +531,12 @@ mod tests { &source, ); - assert!( - records_for(target) - .iter() - .any(|record| record.message.contains("after-poison")) + assert_eq!( + records_for(target), + [LogRecord { + level: Level::Info, + text: "after-poison".to_owned(), + }] ); } } diff --git a/llama-cpp-ffi-status/src/read_and_free_cpp_string.rs b/llama-cpp-ffi-status/src/read_and_free_cpp_string.rs index ab25f7d20..363cf2df1 100644 --- a/llama-cpp-ffi-status/src/read_and_free_cpp_string.rs +++ b/llama-cpp-ffi-status/src/read_and_free_cpp_string.rs @@ -57,8 +57,12 @@ mod tests { }; assert_eq!( - message.map(|text| text.contains("parse error")), - Ok(true), + message, + Ok( + "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error \ + while parsing value - invalid literal; last read: 'no'" + .to_owned() + ), "the llama.cpp json parser reports its failure through the error slot" ); } diff --git a/llama-cpp-test-harness-macros/src/lib.rs b/llama-cpp-test-harness-macros/src/lib.rs index 04b38d8fb..4b1d6b59a 100644 --- a/llama-cpp-test-harness-macros/src/lib.rs +++ b/llama-cpp-test-harness-macros/src/lib.rs @@ -32,6 +32,7 @@ mod tests { use quote::quote; use super::dispatch; + use crate::expand::expand; #[test] fn dispatch_on_invalid_attribute_emits_compile_error_tokens() { @@ -39,11 +40,13 @@ mod tests { let item = quote! { fn my_test(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } }; - let emitted = dispatch(attribute, item).to_string(); + let message = "unknown field `totally_wrong`; expected one of: model_source, \ + n_gpu_layers, load_mode, n_ctx, n_batch, n_ubatch, mmproj_source, \ + embeddings, n_seq_max, n_threads_batch, void_logs"; - assert!( - emitted.contains("compile_error"), - "expected compile_error! tokens in emitted output: {emitted}", + assert_eq!( + dispatch(attribute, item).to_string(), + quote! { ::core::compile_error! { #message } }.to_string() ); } @@ -60,15 +63,8 @@ mod tests { let item = quote! { fn my_test(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } }; - let emitted = dispatch(attribute, item).to_string(); + let expanded = expand(attribute.clone(), item.clone()).expect("valid input must expand"); - assert!( - emitted.contains("inventory"), - "expected inventory::submit in emitted output: {emitted}", - ); - assert!( - !emitted.contains("compile_error"), - "valid input should not emit compile_error: {emitted}", - ); + assert_eq!(dispatch(attribute, item).to_string(), expanded.to_string()); } } diff --git a/llama-cpp-test-harness-macros/src/parsed_args.rs b/llama-cpp-test-harness-macros/src/parsed_args.rs index e262e64cc..7b7c8ba7b 100644 --- a/llama-cpp-test-harness-macros/src/parsed_args.rs +++ b/llama-cpp-test-harness-macros/src/parsed_args.rs @@ -350,10 +350,7 @@ mod tests { .expect_err("missing model_source must fail") .to_string(); - assert!( - message.contains("missing required field `model_source`"), - "got: {message}" - ); + assert_eq!(message, "missing required field `model_source`"); } #[test] @@ -363,10 +360,7 @@ mod tests { .expect_err("missing n_ctx must fail") .to_string(); - assert!( - message.contains("missing required field `n_ctx`"), - "got: {message}" - ); + assert_eq!(message, "missing required field `n_ctx`"); } #[test] @@ -377,7 +371,11 @@ mod tests { .expect_err("legacy repo must be rejected") .to_string(); - assert!(message.contains("model_source"), "got: {message}"); + assert_eq!( + message, + "field `repo` was removed; use `model_source = HuggingFace(repo, file)` or \ + `model_source = LocalPath(path)` (and `mmproj_source` for mmproj)" + ); } #[test] @@ -387,7 +385,11 @@ mod tests { .expect_err("legacy mmproj_file must be rejected") .to_string(); - assert!(message.contains("mmproj_source"), "got: {message}"); + assert_eq!( + message, + "field `mmproj_file` was removed; use `model_source = HuggingFace(repo, file)` or \ + `model_source = LocalPath(path)` (and `mmproj_source` for mmproj)" + ); } #[test] @@ -397,9 +399,11 @@ mod tests { .expect_err("unknown field must fail") .to_string(); - assert!( - message.contains("unknown field `surprise`"), - "got: {message}" + assert_eq!( + message, + "unknown field `surprise`; expected one of: model_source, n_gpu_layers, load_mode, \ + n_ctx, n_batch, n_ubatch, mmproj_source, embeddings, n_seq_max, n_threads_batch, \ + void_logs" ); } @@ -408,10 +412,7 @@ mod tests { let source = format!("{ALL_REQUIRED}, model_source = HuggingFace(\"other\", \"o.gguf\")"); let message = parse(&source).expect_err("duplicate must fail").to_string(); - assert!( - message.contains("duplicate field `model_source`"), - "got: {message}" - ); + assert_eq!(message, "duplicate field `model_source`"); } #[test] @@ -419,7 +420,7 @@ mod tests { let source = "model_source, file = \"x\""; let message = parse(source).expect_err("bare ident must fail").to_string(); - assert!(message.contains("name = value"), "got: {message}"); + assert_eq!(message, "expected `name = value` form"); } #[test] @@ -435,7 +436,7 @@ mod tests { .expect_err("non-literal value must fail") .to_string(); - assert!(message.contains("literal"), "got: {message}"); + assert_eq!(message, "expected a literal (string, integer, or bool)"); } #[test] @@ -451,7 +452,7 @@ mod tests { .expect_err("string for int field must fail") .to_string(); - assert!(message.contains("integer literal"), "got: {message}"); + assert_eq!(message, "field `n_gpu_layers` expects an integer literal"); } #[test] @@ -467,7 +468,7 @@ mod tests { .expect_err("literal load mode must fail") .to_string(); - assert!(message.contains("load-mode identifier"), "got: {message}"); + assert_eq!(message, "expected a load-mode identifier"); } #[test] @@ -483,7 +484,7 @@ mod tests { .expect_err("qualified load mode must fail") .to_string(); - assert!(message.contains("load-mode identifier"), "got: {message}"); + assert_eq!(message, "expected a load-mode identifier"); } #[test] @@ -499,7 +500,7 @@ mod tests { .expect_err("negative int must fail") .to_string(); - assert!(message.contains("literal"), "got: {message}"); + assert_eq!(message, "expected a literal (string, integer, or bool)"); } #[test] @@ -516,7 +517,7 @@ mod tests { .expect_err("path field name must fail") .to_string(); - assert!(message.contains("simple identifier"), "got: {message}"); + assert_eq!(message, "expected a simple identifier"); } #[test] @@ -530,7 +531,11 @@ mod tests { n_ubatch = 1"; let message = parse(source).expect_err("overflow must fail").to_string(); - assert!(message.contains("u32"), "got: {message}"); + assert_eq!( + message, + "field `n_ctx` expects a non-negative integer that fits in u32: \ + number too large to fit in target type" + ); } #[test] @@ -540,7 +545,11 @@ mod tests { .expect_err("i32 overflow must fail") .to_string(); - assert!(message.contains("i32"), "got: {message}"); + assert_eq!( + message, + "field `n_threads_batch` expects an integer that fits in i32: \ + number too large to fit in target type" + ); } #[test] @@ -555,10 +564,7 @@ mod tests { .expect_err("missing n_gpu_layers must fail") .to_string(); - assert!( - message.contains("missing required field `n_gpu_layers`"), - "got: {message}" - ); + assert_eq!(message, "missing required field `n_gpu_layers`"); } #[test] @@ -573,10 +579,7 @@ mod tests { .expect_err("missing load_mode must fail") .to_string(); - assert!( - message.contains("missing required field `load_mode`"), - "got: {message}" - ); + assert_eq!(message, "missing required field `load_mode`"); } #[test] @@ -592,7 +595,10 @@ mod tests { .expect_err("unknown load mode must fail") .to_string(); - assert!(message.contains("MmapMlock"), "got: {message}"); + assert_eq!( + message, + "expected one of: Auto, None, Mmap, Mlock, MmapMlock, DirectIo" + ); } #[test] @@ -607,10 +613,7 @@ mod tests { .expect_err("missing n_batch must fail") .to_string(); - assert!( - message.contains("missing required field `n_batch`"), - "got: {message}" - ); + assert_eq!(message, "missing required field `n_batch`"); } #[test] @@ -625,10 +628,7 @@ mod tests { .expect_err("missing n_ubatch must fail") .to_string(); - assert!( - message.contains("missing required field `n_ubatch`"), - "got: {message}" - ); + assert_eq!(message, "missing required field `n_ubatch`"); } #[test] @@ -661,7 +661,10 @@ mod tests { .expect_err("embeddings with int must fail") .to_string(); - assert!(message.contains("bool literal"), "got: {message}"); + assert_eq!( + message, + "field `embeddings` expects a bool literal (`true` or `false`)" + ); } #[test] @@ -701,7 +704,10 @@ mod tests { .expect_err("string for n_threads_batch must fail") .to_string(); - assert!(message.contains("integer literal"), "got: {message}"); + assert_eq!( + message, + "field `n_threads_batch` expects an integer literal" + ); } #[test] @@ -748,7 +754,10 @@ mod tests { let source = override_field(field, "some_const"); let message = parse(&source).expect_err(field).to_string(); - assert!(message.contains("literal"), "{field}: {message}"); + assert_eq!( + message, "expected a literal (string, integer, or bool)", + "{field}" + ); } } @@ -758,7 +767,10 @@ mod tests { let source = override_field(field, "\"not-an-int\""); let message = parse(&source).expect_err(field).to_string(); - assert!(message.contains("integer literal"), "{field}: {message}"); + assert_eq!( + message, + format!("field `{field}` expects an integer literal") + ); } } @@ -769,7 +781,7 @@ mod tests { .expect_err("n_seq_max non-literal must fail") .to_string(); - assert!(message.contains("literal"), "got: {message}"); + assert_eq!(message, "expected a literal (string, integer, or bool)"); } #[test] @@ -779,7 +791,7 @@ mod tests { .expect_err("n_seq_max wrong-kind must fail") .to_string(); - assert!(message.contains("integer literal"), "got: {message}"); + assert_eq!(message, "field `n_seq_max` expects an integer literal"); } #[test] @@ -789,7 +801,7 @@ mod tests { .expect_err("n_threads_batch non-literal must fail") .to_string(); - assert!(message.contains("literal"), "got: {message}"); + assert_eq!(message, "expected a literal (string, integer, or bool)"); } #[test] @@ -799,7 +811,7 @@ mod tests { .expect_err("embeddings non-literal must fail") .to_string(); - assert!(message.contains("literal"), "got: {message}"); + assert_eq!(message, "expected a literal (string, integer, or bool)"); } #[test] @@ -809,7 +821,7 @@ mod tests { .expect_err("void_logs non-literal must fail") .to_string(); - assert!(message.contains("literal"), "got: {message}"); + assert_eq!(message, "expected a literal (string, integer, or bool)"); } #[test] @@ -819,7 +831,10 @@ mod tests { .expect_err("void_logs wrong-kind must fail") .to_string(); - assert!(message.contains("bool literal"), "got: {message}"); + assert_eq!( + message, + "field `void_logs` expects a bool literal (`true` or `false`)" + ); } #[test] @@ -829,7 +844,10 @@ mod tests { .expect_err("mmproj_source unknown variant must fail") .to_string(); - assert!(message.contains("unknown source variant"), "got: {message}"); + assert_eq!( + message, + "unknown source variant `Mystery`; expected `HuggingFace` or `LocalPath`" + ); } #[test] @@ -845,7 +863,10 @@ mod tests { .expect_err("model_source unknown variant must fail") .to_string(); - assert!(message.contains("unknown source variant"), "got: {message}"); + assert_eq!( + message, + "unknown source variant `Mystery`; expected `HuggingFace` or `LocalPath`" + ); } #[test] diff --git a/llama-cpp-test-harness-macros/src/parsed_load_mode.rs b/llama-cpp-test-harness-macros/src/parsed_load_mode.rs index 1f9773ffc..9cac56011 100644 --- a/llama-cpp-test-harness-macros/src/parsed_load_mode.rs +++ b/llama-cpp-test-harness-macros/src/parsed_load_mode.rs @@ -77,8 +77,9 @@ mod tests { .expect_err("unknown mode must fail") .to_string(); - for valid in ["Auto", "None", "Mmap", "Mlock", "MmapMlock", "DirectIo"] { - assert!(message.contains(valid), "missing {valid} in {message}"); - } + assert_eq!( + message, + "expected one of: Auto, None, Mmap, Mlock, MmapMlock, DirectIo" + ); } } diff --git a/llama-cpp-test-harness-macros/src/parsed_source.rs b/llama-cpp-test-harness-macros/src/parsed_source.rs index bba3fa031..fd3451806 100644 --- a/llama-cpp-test-harness-macros/src/parsed_source.rs +++ b/llama-cpp-test-harness-macros/src/parsed_source.rs @@ -137,7 +137,10 @@ mod tests { .expect_err("unknown variant must fail") .to_string(); - assert!(message.contains("unknown source variant"), "got: {message}"); + assert_eq!( + message, + "unknown source variant `Mystery`; expected `HuggingFace` or `LocalPath`" + ); } #[test] @@ -146,8 +149,10 @@ mod tests { .expect_err("non-call must fail") .to_string(); - assert!(message.contains("HuggingFace"), "got: {message}"); - assert!(message.contains("LocalPath"), "got: {message}"); + assert_eq!( + message, + "field `model_source` expects `HuggingFace(repo, file)` or `LocalPath(path)`" + ); } #[test] @@ -156,8 +161,10 @@ mod tests { .expect_err("arity must fail") .to_string(); - assert!(message.contains("HuggingFace"), "got: {message}"); - assert!(message.contains("2 string"), "got: {message}"); + assert_eq!( + message, + "`HuggingFace` expects exactly 2 string arguments (repo, file); got 1" + ); } #[test] @@ -166,8 +173,10 @@ mod tests { .expect_err("arity must fail") .to_string(); - assert!(message.contains("LocalPath"), "got: {message}"); - assert!(message.contains("1 string"), "got: {message}"); + assert_eq!( + message, + "`LocalPath` expects exactly 1 string argument (path); got 2" + ); } #[test] @@ -176,7 +185,10 @@ mod tests { .expect_err("non-string arg must fail") .to_string(); - assert!(message.contains("string literal"), "got: {message}"); + assert_eq!( + message, + "`model_source` argument `HuggingFace.repo` expects a string literal" + ); } #[test] @@ -185,7 +197,10 @@ mod tests { .expect_err("non-string second arg must fail") .to_string(); - assert!(message.contains("string literal"), "got: {message}"); + assert_eq!( + message, + "`model_source` argument `HuggingFace.file` expects a string literal" + ); } #[test] @@ -194,7 +209,10 @@ mod tests { .expect_err("non-string LocalPath arg must fail") .to_string(); - assert!(message.contains("string literal"), "got: {message}"); + assert_eq!( + message, + "`model_source` argument `LocalPath.path` expects a string literal" + ); } #[test] @@ -213,7 +231,10 @@ mod tests { .expect_err("non-path func must fail") .to_string(); - assert!(message.contains("HuggingFace"), "got: {message}"); + assert_eq!( + message, + "field `model_source` expects `HuggingFace(...)` or `LocalPath(...)`" + ); } #[test] @@ -222,7 +243,10 @@ mod tests { .expect_err("qualified path must fail") .to_string(); - assert!(message.contains("bare variant name"), "got: {message}"); + assert_eq!( + message, + "field `model_source` expects the bare variant name `HuggingFace` or `LocalPath`" + ); } #[test] From b47f827b4bdf4bbdfb8e0a766b040b7d7ed226ad Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Fri, 30 Oct 2026 14:11:23 +0100 Subject: [PATCH 07/82] Compile and lint the wrappers against llama.cpp and GSL as system headers so vendored code no longer warns --- .../src/bindgen_config.rs | 1 + llama-cpp-bindings-build/src/cpp_wrapper.rs | 5 +++ llama-cpp-bindings-sys/wrapper.h | 4 +- llama-cpp-bindings-sys/wrapper_batch.cpp | 2 +- llama-cpp-bindings-sys/wrapper_batch.h | 2 +- .../wrapper_chat_session.cpp | 12 +++--- llama-cpp-bindings-sys/wrapper_chat_session.h | 2 +- .../wrapper_chat_templates.cpp | 12 +++--- .../wrapper_chat_templates.h | 2 +- .../wrapper_chat_templates_handle.h | 2 +- llama-cpp-bindings-sys/wrapper_common.cpp | 6 +-- llama-cpp-bindings-sys/wrapper_common.h | 2 +- llama-cpp-bindings-sys/wrapper_context.cpp | 10 ++--- llama-cpp-bindings-sys/wrapper_fit.cpp | 6 +-- llama-cpp-bindings-sys/wrapper_fit.h | 4 +- llama-cpp-bindings-sys/wrapper_gbnf.cpp | 2 +- llama-cpp-bindings-sys/wrapper_mtmd.cpp | 6 +-- llama-cpp-bindings-sys/wrapper_mtmd.h | 6 +-- llama-cpp-bindings-sys/wrapper_parsed_chat.h | 2 +- .../wrapper_parsed_chat_handle.h | 2 +- .../wrapper_sampler_init.cpp | 2 +- llama-cpp-bindings-sys/wrapper_sampler_init.h | 2 +- llama-cpp-bindings-sys/wrapper_state.cpp | 2 +- llama-cpp-bindings-sys/wrapper_state.h | 2 +- .../src/compile_commands_file.rs | 37 +++++++++++++++++++ llama-cpp-wrapper-sources/src/lib.rs | 1 + .../src/vendored_include_dirs.rs | 8 ++++ .../src/wrapper_include_dirs.rs | 10 +---- 28 files changed, 99 insertions(+), 55 deletions(-) create mode 100644 llama-cpp-wrapper-sources/src/vendored_include_dirs.rs diff --git a/llama-cpp-bindings-build/src/bindgen_config.rs b/llama-cpp-bindings-build/src/bindgen_config.rs index ca38fff00..40fd9dc67 100644 --- a/llama-cpp-bindings-build/src/bindgen_config.rs +++ b/llama-cpp-bindings-build/src/bindgen_config.rs @@ -184,6 +184,7 @@ pub fn generate_bindings( fn create_base_builder(llama_src: &Path, callbacks: BindingCallbacks) -> bindgen::Builder { let mut builder = bindgen::Builder::default() .header("wrapper.h") + .clang_arg(format!("-I{}", llama_src.display())) .clang_arg(format!("-I{}", llama_src.join("include").display())) .clang_arg(format!("-I{}", llama_src.join("ggml/include").display())) .parse_callbacks(Box::new(callbacks)) diff --git a/llama-cpp-bindings-build/src/cpp_wrapper.rs b/llama-cpp-bindings-build/src/cpp_wrapper.rs index 2c1bc17c7..d44afddcc 100644 --- a/llama-cpp-bindings-build/src/cpp_wrapper.rs +++ b/llama-cpp-bindings-build/src/cpp_wrapper.rs @@ -1,4 +1,5 @@ use llama_cpp_wrapper_sources::cpp_standard::CPP_STANDARD; +use llama_cpp_wrapper_sources::vendored_include_dirs::VENDORED_INCLUDE_DIRS; use llama_cpp_wrapper_sources::wrapper_include_dirs::WRAPPER_INCLUDE_DIRS; use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; @@ -14,6 +15,10 @@ pub fn compile_cpp_wrappers(target_os: TargetOs) -> Result<(), BuildError> { build.include(include_dir); } + for include_dir in VENDORED_INCLUDE_DIRS { + build.flag(format!("-isystem{include_dir}")); + } + build.flag_if_supported(format!("-std={CPP_STANDARD}")); for source in WRAPPER_SOURCES { diff --git a/llama-cpp-bindings-sys/wrapper.h b/llama-cpp-bindings-sys/wrapper.h index c7c0853d8..c9c12e8b1 100644 --- a/llama-cpp-bindings-sys/wrapper.h +++ b/llama-cpp-bindings-sys/wrapper.h @@ -1,5 +1,5 @@ -#include "llama.cpp/include/llama.h" -#include "llama.cpp/ggml/include/gguf.h" +#include +#include #include "wrapper_batch.h" #include "wrapper_chat_session.h" #include "wrapper_chat_templates.h" diff --git a/llama-cpp-bindings-sys/wrapper_batch.cpp b/llama-cpp-bindings-sys/wrapper_batch.cpp index 0d2465ae5..ef7722234 100644 --- a/llama-cpp-bindings-sys/wrapper_batch.cpp +++ b/llama-cpp-bindings-sys/wrapper_batch.cpp @@ -1,7 +1,7 @@ #include "wrapper_batch.h" #include "wrapper_report_current_exception.h" -#include "llama.cpp/include/llama.h" +#include #include #include diff --git a/llama-cpp-bindings-sys/wrapper_batch.h b/llama-cpp-bindings-sys/wrapper_batch.h index 660fde705..396aa9468 100644 --- a/llama-cpp-bindings-sys/wrapper_batch.h +++ b/llama-cpp-bindings-sys/wrapper_batch.h @@ -1,6 +1,6 @@ #pragma once -#include "llama.cpp/include/llama.h" +#include #include #include diff --git a/llama-cpp-bindings-sys/wrapper_chat_session.cpp b/llama-cpp-bindings-sys/wrapper_chat_session.cpp index 4efee6f96..406cbff71 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_session.cpp +++ b/llama-cpp-bindings-sys/wrapper_chat_session.cpp @@ -6,12 +6,12 @@ #include "wrapper_report_current_exception.h" #include "wrapper_utils.h" -#include "llama.cpp/common/chat-peg-parser.h" -#include "llama.cpp/common/chat.h" -#include "llama.cpp/common/common.h" -#include "llama.cpp/common/json.h" -#include "llama.cpp/common/peg-parser.h" -#include "llama.cpp/include/llama.h" +#include +#include +#include +#include +#include +#include #include #include diff --git a/llama-cpp-bindings-sys/wrapper_chat_session.h b/llama-cpp-bindings-sys/wrapper_chat_session.h index 1bfd36354..6ea75b3e3 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_session.h +++ b/llama-cpp-bindings-sys/wrapper_chat_session.h @@ -1,6 +1,6 @@ #pragma once -#include "llama.cpp/include/llama.h" +#include #include "wrapper_parsed_chat.h" #include "wrapper_chat_templates.h" diff --git a/llama-cpp-bindings-sys/wrapper_chat_templates.cpp b/llama-cpp-bindings-sys/wrapper_chat_templates.cpp index dddd021e1..64809ccad 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_templates.cpp +++ b/llama-cpp-bindings-sys/wrapper_chat_templates.cpp @@ -2,12 +2,12 @@ #include "wrapper_chat_templates_handle.h" #include "wrapper_report_current_exception.h" -#include "llama.cpp/common/chat.h" -#include "llama.cpp/common/jinja/caps.h" -#include "llama.cpp/common/jinja/lexer.h" -#include "llama.cpp/common/jinja/parser.h" -#include "llama.cpp/common/jinja/runtime.h" -#include "llama.cpp/include/llama.h" +#include +#include +#include +#include +#include +#include #include #include diff --git a/llama-cpp-bindings-sys/wrapper_chat_templates.h b/llama-cpp-bindings-sys/wrapper_chat_templates.h index 9533a0062..bdaa5ca88 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_templates.h +++ b/llama-cpp-bindings-sys/wrapper_chat_templates.h @@ -1,6 +1,6 @@ #pragma once -#include "llama.cpp/include/llama.h" +#include #include diff --git a/llama-cpp-bindings-sys/wrapper_chat_templates_handle.h b/llama-cpp-bindings-sys/wrapper_chat_templates_handle.h index eaf108db0..69d8bbb12 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_templates_handle.h +++ b/llama-cpp-bindings-sys/wrapper_chat_templates_handle.h @@ -1,6 +1,6 @@ #pragma once -#include "llama.cpp/common/chat.h" +#include struct llama_rs_chat_templates { common_chat_templates_ptr templates; diff --git a/llama-cpp-bindings-sys/wrapper_common.cpp b/llama-cpp-bindings-sys/wrapper_common.cpp index ada517d0a..50d355819 100644 --- a/llama-cpp-bindings-sys/wrapper_common.cpp +++ b/llama-cpp-bindings-sys/wrapper_common.cpp @@ -8,9 +8,9 @@ #include #include -#include "llama.cpp/common/json-schema-to-grammar.h" -#include "llama.cpp/common/json.h" -#include "llama.cpp/include/llama.h" +#include +#include +#include #include "wrapper_report_current_exception.h" #include "wrapper_utils.h" diff --git a/llama-cpp-bindings-sys/wrapper_common.h b/llama-cpp-bindings-sys/wrapper_common.h index f99bf9056..9bae6830e 100644 --- a/llama-cpp-bindings-sys/wrapper_common.h +++ b/llama-cpp-bindings-sys/wrapper_common.h @@ -1,6 +1,6 @@ #pragma once -#include "llama.cpp/include/llama.h" +#include #include #include diff --git a/llama-cpp-bindings-sys/wrapper_context.cpp b/llama-cpp-bindings-sys/wrapper_context.cpp index 825e6a39e..0da34bc02 100644 --- a/llama-cpp-bindings-sys/wrapper_context.cpp +++ b/llama-cpp-bindings-sys/wrapper_context.cpp @@ -2,11 +2,11 @@ #include -#include "llama.cpp/include/llama.h" -#include "llama.cpp/src/llama-context.h" -#include "llama.cpp/src/llama-ext.h" -#include "llama.cpp/src/llama-memory.h" -#include "llama.cpp/src/llama-model.h" +#include +#include +#include +#include +#include #include "wrapper_report_current_exception.h" extern "C" auto llama_rs_context_embeddings_enabled(const struct llama_context * ctx) -> bool { diff --git a/llama-cpp-bindings-sys/wrapper_fit.cpp b/llama-cpp-bindings-sys/wrapper_fit.cpp index 7cb39e651..66b27798b 100644 --- a/llama-cpp-bindings-sys/wrapper_fit.cpp +++ b/llama-cpp-bindings-sys/wrapper_fit.cpp @@ -1,12 +1,12 @@ #include "wrapper_fit.h" -#include "llama.h" -#include "ggml.h" +#include +#include #include "wrapper_report_current_exception.h" #include #include -#include "llama.cpp/common/fit.h" +#include extern "C" auto llama_rs_fit_params( const char * path_model, diff --git a/llama-cpp-bindings-sys/wrapper_fit.h b/llama-cpp-bindings-sys/wrapper_fit.h index f9485cde6..514edba17 100644 --- a/llama-cpp-bindings-sys/wrapper_fit.h +++ b/llama-cpp-bindings-sys/wrapper_fit.h @@ -1,7 +1,7 @@ #pragma once -#include "llama.cpp/ggml/include/ggml.h" -#include "llama.cpp/include/llama.h" +#include +#include #include #include diff --git a/llama-cpp-bindings-sys/wrapper_gbnf.cpp b/llama-cpp-bindings-sys/wrapper_gbnf.cpp index 5fc9f4e90..644f497af 100644 --- a/llama-cpp-bindings-sys/wrapper_gbnf.cpp +++ b/llama-cpp-bindings-sys/wrapper_gbnf.cpp @@ -1,7 +1,7 @@ #include "wrapper_gbnf.h" #include "wrapper_report_current_exception.h" -#include "llama.cpp/src/llama-grammar.h" +#include extern "C" auto llama_rs_validate_gbnf( diff --git a/llama-cpp-bindings-sys/wrapper_mtmd.cpp b/llama-cpp-bindings-sys/wrapper_mtmd.cpp index b56a0fb78..04f806ab0 100644 --- a/llama-cpp-bindings-sys/wrapper_mtmd.cpp +++ b/llama-cpp-bindings-sys/wrapper_mtmd.cpp @@ -1,7 +1,7 @@ #include "wrapper_mtmd.h" -#include "llama.h" -#include "tools/mtmd/mtmd.h" -#include "tools/mtmd/mtmd-helper.h" +#include +#include +#include #include "wrapper_report_current_exception.h" #include diff --git a/llama-cpp-bindings-sys/wrapper_mtmd.h b/llama-cpp-bindings-sys/wrapper_mtmd.h index 192d7a237..80b774bb0 100644 --- a/llama-cpp-bindings-sys/wrapper_mtmd.h +++ b/llama-cpp-bindings-sys/wrapper_mtmd.h @@ -1,8 +1,8 @@ #pragma once -#include "llama.cpp/include/llama.h" -#include "llama.cpp/tools/mtmd/mtmd.h" -#include "llama.cpp/tools/mtmd/mtmd-helper.h" +#include +#include +#include #include #include diff --git a/llama-cpp-bindings-sys/wrapper_parsed_chat.h b/llama-cpp-bindings-sys/wrapper_parsed_chat.h index 6cd20e6c0..d7bc5f1bf 100644 --- a/llama-cpp-bindings-sys/wrapper_parsed_chat.h +++ b/llama-cpp-bindings-sys/wrapper_parsed_chat.h @@ -1,6 +1,6 @@ #pragma once -#include "llama.cpp/include/llama.h" +#include #include diff --git a/llama-cpp-bindings-sys/wrapper_parsed_chat_handle.h b/llama-cpp-bindings-sys/wrapper_parsed_chat_handle.h index 976d9c647..b6b318a86 100644 --- a/llama-cpp-bindings-sys/wrapper_parsed_chat_handle.h +++ b/llama-cpp-bindings-sys/wrapper_parsed_chat_handle.h @@ -1,6 +1,6 @@ #pragma once -#include "llama.cpp/common/chat.h" +#include struct llama_rs_parsed_chat { common_chat_msg message; diff --git a/llama-cpp-bindings-sys/wrapper_sampler_init.cpp b/llama-cpp-bindings-sys/wrapper_sampler_init.cpp index ee26f71bc..ecadf36bb 100644 --- a/llama-cpp-bindings-sys/wrapper_sampler_init.cpp +++ b/llama-cpp-bindings-sys/wrapper_sampler_init.cpp @@ -1,6 +1,6 @@ #include "wrapper_sampler_init.h" -#include "llama.cpp/include/llama.h" +#include #include "wrapper_report_current_exception.h" #include diff --git a/llama-cpp-bindings-sys/wrapper_sampler_init.h b/llama-cpp-bindings-sys/wrapper_sampler_init.h index dbf4ac229..2f7abfcb2 100644 --- a/llama-cpp-bindings-sys/wrapper_sampler_init.h +++ b/llama-cpp-bindings-sys/wrapper_sampler_init.h @@ -1,6 +1,6 @@ #pragma once -#include "llama.cpp/include/llama.h" +#include #include #include diff --git a/llama-cpp-bindings-sys/wrapper_state.cpp b/llama-cpp-bindings-sys/wrapper_state.cpp index 0fef5235b..230ae53ad 100644 --- a/llama-cpp-bindings-sys/wrapper_state.cpp +++ b/llama-cpp-bindings-sys/wrapper_state.cpp @@ -1,6 +1,6 @@ #include "wrapper_state.h" -#include "llama.cpp/include/llama.h" +#include #include "wrapper_report_current_exception.h" #include diff --git a/llama-cpp-bindings-sys/wrapper_state.h b/llama-cpp-bindings-sys/wrapper_state.h index 4f76fe630..199e3ccd1 100644 --- a/llama-cpp-bindings-sys/wrapper_state.h +++ b/llama-cpp-bindings-sys/wrapper_state.h @@ -1,6 +1,6 @@ #pragma once -#include "llama.cpp/include/llama.h" +#include #include #include diff --git a/llama-cpp-wrapper-sources/src/compile_commands_file.rs b/llama-cpp-wrapper-sources/src/compile_commands_file.rs index 2744876f5..62bf71bd3 100644 --- a/llama-cpp-wrapper-sources/src/compile_commands_file.rs +++ b/llama-cpp-wrapper-sources/src/compile_commands_file.rs @@ -4,6 +4,7 @@ use std::path::PathBuf; use crate::compile_command::CompileCommand; use crate::cpp_standard::CPP_STANDARD; +use crate::vendored_include_dirs::VENDORED_INCLUDE_DIRS; use crate::wrapper_include_dirs::WRAPPER_INCLUDE_DIRS; use crate::wrapper_source_paths::wrapper_source_paths; use crate::wrapper_sources_error::WrapperSourcesError; @@ -31,6 +32,13 @@ impl CompileCommandsFile { arguments.push(format!("-I{}", absolute_path(&self.sys_dir, include_dir))); } + for include_dir in VENDORED_INCLUDE_DIRS { + arguments.push(format!( + "-isystem{}", + absolute_path(&self.sys_dir, include_dir) + )); + } + arguments.push("-c".to_owned()); arguments.push(file.display().to_string()); @@ -69,6 +77,7 @@ mod tests { use std::path::PathBuf; use super::CompileCommandsFile; + use crate::vendored_include_dirs::VENDORED_INCLUDE_DIRS; use crate::wrapper_include_dirs::WRAPPER_INCLUDE_DIRS; use crate::wrapper_source_paths::wrapper_source_paths; use crate::wrapper_sources_error::WrapperSourcesError; @@ -121,6 +130,34 @@ mod tests { ); } + #[test] + fn every_vendored_include_dir_is_a_system_include_resolved_against_the_sys_dir() { + let commands = database_for("/unused").commands(); + + assert_eq!( + commands + .first() + .map(|command| command + .arguments + .iter() + .filter(|argument| argument.starts_with("-isystem")) + .cloned() + .collect::>()) + .unwrap_or_default(), + VENDORED_INCLUDE_DIRS + .iter() + .map(|include_dir| { + format!( + "-isystem{}", + Path::new("/repo/llama-cpp-bindings-sys") + .join(include_dir) + .display() + ) + }) + .collect::>() + ); + } + #[test] fn the_compiler_and_standard_lead_the_command_line() { let commands = database_for("/unused").commands(); diff --git a/llama-cpp-wrapper-sources/src/lib.rs b/llama-cpp-wrapper-sources/src/lib.rs index 92533a21f..512649a46 100644 --- a/llama-cpp-wrapper-sources/src/lib.rs +++ b/llama-cpp-wrapper-sources/src/lib.rs @@ -11,6 +11,7 @@ pub mod compile_command; pub mod compile_commands_file; pub mod cpp_standard; +pub mod vendored_include_dirs; pub mod wrapper_headers; pub mod wrapper_include_dirs; pub mod wrapper_source_paths; diff --git a/llama-cpp-wrapper-sources/src/vendored_include_dirs.rs b/llama-cpp-wrapper-sources/src/vendored_include_dirs.rs new file mode 100644 index 000000000..ce8eb6c54 --- /dev/null +++ b/llama-cpp-wrapper-sources/src/vendored_include_dirs.rs @@ -0,0 +1,8 @@ +pub const VENDORED_INCLUDE_DIRS: &[&str] = &[ + "GSL/include", + "llama.cpp", + "llama.cpp/common", + "llama.cpp/include", + "llama.cpp/ggml/include", + "llama.cpp/vendor", +]; diff --git a/llama-cpp-wrapper-sources/src/wrapper_include_dirs.rs b/llama-cpp-wrapper-sources/src/wrapper_include_dirs.rs index 732fb83a8..316b56433 100644 --- a/llama-cpp-wrapper-sources/src/wrapper_include_dirs.rs +++ b/llama-cpp-wrapper-sources/src/wrapper_include_dirs.rs @@ -1,9 +1 @@ -pub const WRAPPER_INCLUDE_DIRS: &[&str] = &[ - ".", - "GSL/include", - "llama.cpp", - "llama.cpp/common", - "llama.cpp/include", - "llama.cpp/ggml/include", - "llama.cpp/vendor", -]; +pub const WRAPPER_INCLUDE_DIRS: &[&str] = &["."]; From 6674acb3914c6c04904abde79e4aa257ce10fbb4 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Fri, 30 Oct 2026 14:11:26 +0100 Subject: [PATCH 08/82] Measure the multimodal projector's memory, reserve it in the fit margins, and cap non-causal image tokens to the micro-batch --- llama-cpp-bindings-sys/wrapper_mtmd.cpp | 58 +++ llama-cpp-bindings-sys/wrapper_mtmd.h | 28 ++ .../tests/multimodal_projector_loading.rs | 41 +- .../tests/parameter_fitting.rs | 57 ++- llama-cpp-bindings/src/mtmd.rs | 8 + llama-cpp-bindings/src/mtmd/mtmd_context.rs | 2 +- .../src/mtmd/mtmd_context_params.rs | 95 +++- .../src/mtmd/mtmd_context_params_error.rs | 19 + .../src/mtmd/mtmd_image_token_limit.rs | 98 ++++ .../src/mtmd/mtmd_init_error.rs | 4 + .../src/mtmd/mtmd_memory_usage.rs | 424 ++++++++++++++++++ .../src/mtmd/mtmd_memory_usage_error.rs | 21 + 12 files changed, 838 insertions(+), 17 deletions(-) create mode 100644 llama-cpp-bindings/src/mtmd/mtmd_context_params_error.rs create mode 100644 llama-cpp-bindings/src/mtmd/mtmd_image_token_limit.rs create mode 100644 llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs create mode 100644 llama-cpp-bindings/src/mtmd/mtmd_memory_usage_error.rs diff --git a/llama-cpp-bindings-sys/wrapper_mtmd.cpp b/llama-cpp-bindings-sys/wrapper_mtmd.cpp index 04f806ab0..33ff45148 100644 --- a/llama-cpp-bindings-sys/wrapper_mtmd.cpp +++ b/llama-cpp-bindings-sys/wrapper_mtmd.cpp @@ -309,3 +309,61 @@ extern "C" auto llama_rs_mtmd_eval_chunk_single( LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_ERROR_STRING_ALLOCATION_FAILED); } } + +extern "C" auto llama_rs_mtmd_get_memory_usage( + const char * mmproj_path, + struct mtmd_context_params ctx_params, + llama_rs_mtmd_device_memory * out_device_memory, + size_t device_memory_capacity, + size_t * out_device_memory_count, + int32_t * out_image_max_tokens, + bool * out_use_non_causal, + char ** out_error) -> llama_rs_mtmd_get_memory_usage_status { + if (out_error != nullptr) { + *out_error = nullptr; + } + if (mmproj_path == nullptr) { + return LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_MMPROJ_PATH_ARG; + } + if (out_device_memory == nullptr) { + return LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_DEVICE_MEMORY_ARG; + } + if (out_device_memory_count == nullptr) { + return LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_DEVICE_MEMORY_COUNT_ARG; + } + if (out_image_max_tokens == nullptr) { + return LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_IMAGE_MAX_TOKENS_ARG; + } + if (out_use_non_causal == nullptr) { + return LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_USE_NON_CAUSAL_ARG; + } + *out_device_memory_count = 0; + + try { + const mtmd_memory_usage usage = mtmd_get_memory_usage(mmproj_path, ctx_params); + + if (usage.backend_mem_usage.size() > device_memory_capacity) { + return LLAMA_RS_MTMD_GET_MEMORY_USAGE_DEVICE_MEMORY_CAPACITY_EXCEEDED; + } + + const gsl::span device_memory(out_device_memory, device_memory_capacity); + size_t device_memory_count = 0; + + for (const auto & [device, bytes] : usage.backend_mem_usage) { + device_memory[device_memory_count] = llama_rs_mtmd_device_memory{device, bytes}; + device_memory_count += 1; + } + + *out_device_memory_count = device_memory_count; + *out_image_max_tokens = usage.image_max_tokens; + *out_use_non_causal = usage.use_non_causal; + + return LLAMA_RS_MTMD_GET_MEMORY_USAGE_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_ERROR_STRING_ALLOCATION_FAILED); + } +} diff --git a/llama-cpp-bindings-sys/wrapper_mtmd.h b/llama-cpp-bindings-sys/wrapper_mtmd.h index 80b774bb0..86393586d 100644 --- a/llama-cpp-bindings-sys/wrapper_mtmd.h +++ b/llama-cpp-bindings-sys/wrapper_mtmd.h @@ -150,6 +150,34 @@ llama_rs_mtmd_eval_chunk_single_status llama_rs_mtmd_eval_chunk_single( int32_t * out_llama_cpp_return_code, char ** out_error); +typedef struct llama_rs_mtmd_device_memory { + ggml_backend_dev_t device; + size_t bytes; +} llama_rs_mtmd_device_memory; + +typedef enum llama_rs_mtmd_get_memory_usage_status { + LLAMA_RS_MTMD_GET_MEMORY_USAGE_OK = 0, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_MMPROJ_PATH_ARG, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_DEVICE_MEMORY_ARG, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_DEVICE_MEMORY_COUNT_ARG, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_IMAGE_MAX_TOKENS_ARG, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_USE_NON_CAUSAL_ARG, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_DEVICE_MEMORY_CAPACITY_EXCEEDED, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_mtmd_get_memory_usage_status; + +llama_rs_mtmd_get_memory_usage_status llama_rs_mtmd_get_memory_usage( + const char * mmproj_path, + struct mtmd_context_params ctx_params, + llama_rs_mtmd_device_memory * out_device_memory, + size_t device_memory_capacity, + size_t * out_device_memory_count, + int32_t * out_image_max_tokens, + bool * out_use_non_causal, + char ** out_error); + #ifdef __cplusplus } #endif diff --git a/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs b/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs index 38ec07a55..b93d6fa12 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs @@ -1,5 +1,11 @@ +use std::ffi::CString; +use std::num::NonZeroU32; + use llama_cpp_bindings::mtmd::MtmdContext; use llama_cpp_bindings::mtmd::MtmdContextParams; +use llama_cpp_bindings::mtmd::MtmdContextParamsError; +use llama_cpp_bindings::mtmd::MtmdImageTokenLimit; +use llama_cpp_bindings::mtmd::MtmdInitError; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -23,7 +29,38 @@ use llama_cpp_test_harness::llama_test; )] fn init_from_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_>) { let mtmd_params = MtmdContextParams::default(); - let result = MtmdContext::init_from_file("path\0null", fixture.model, &mtmd_params); - assert!(result.is_err()); + assert_eq!( + MtmdContext::init_from_file("path\0null", fixture.model, &mtmd_params).err(), + CString::new("path\0null") + .err() + .map(MtmdInitError::CStringError) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn init_from_file_refuses_an_image_token_limit_a_c_int_cannot_hold(fixture: &LlamaFixture<'_>) { + let limit = MtmdImageTokenLimit::Tokens(NonZeroU32::MAX); + let mtmd_params = MtmdContextParams { + image_max_tokens: limit, + ..MtmdContextParams::default() + }; + + assert_eq!( + MtmdContext::init_from_file("mmproj.gguf", fixture.model, &mtmd_params).err(), + i32::try_from(u32::MAX).err().map(|source| { + MtmdInitError::ContextParams(MtmdContextParamsError::ImageMaxTokensExceedCInt { + limit, + source, + }) + }) + ); } diff --git a/llama-cpp-bindings-tests/tests/parameter_fitting.rs b/llama-cpp-bindings-tests/tests/parameter_fitting.rs index 13222ad3d..8e600d1f0 100644 --- a/llama-cpp-bindings-tests/tests/parameter_fitting.rs +++ b/llama-cpp-bindings-tests/tests/parameter_fitting.rs @@ -8,6 +8,9 @@ use llama_cpp_bindings::context::params::LlamaContextParams; use llama_cpp_bindings::max_devices; use llama_cpp_bindings::model::params::LlamaModelParams; use llama_cpp_bindings::model::params::fit_extra_model::FitExtraModel; +use llama_cpp_bindings::mtmd::MtmdContextParams; +use llama_cpp_bindings::mtmd::MtmdMemoryUsage; +use llama_cpp_test_harness::GgufSource; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -44,11 +47,7 @@ use llama_cpp_test_harness::llama_test; n_ubatch = 64, )] fn fit_params_succeeds_with_test_model(fixture: &LlamaFixture<'_>) -> Result<()> { - let model_path_utf8 = fixture - .model_path - .to_str() - .ok_or_else(|| anyhow!("model path is not valid UTF-8"))?; - let model_path_c = CString::new(model_path_utf8)?; + let model_path_c = CString::new(fixture.model_path.as_os_str().as_encoded_bytes())?; let mut params = pin!(LlamaModelParams::default()); let mut context_params = LlamaContextParams::default(); @@ -80,11 +79,7 @@ fn fit_params_succeeds_with_test_model(fixture: &LlamaFixture<'_>) -> Result<()> fn fit_params_aligns_the_extra_model_context_with_the_fitted_one( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let model_path_utf8 = fixture - .model_path - .to_str() - .ok_or_else(|| anyhow!("model path is not valid UTF-8"))?; - let model_path_c = CString::new(model_path_utf8)?; + let model_path_c = CString::new(fixture.model_path.as_os_str().as_encoded_bytes())?; let mut params = pin!(LlamaModelParams::default()); let mut context_params = LlamaContextParams::default(); @@ -124,3 +119,45 @@ fn fit_params_aligns_the_extra_model_context_with_the_fitted_one( Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn fit_params_fits_the_model_beside_the_projector_memory_reserved_on_its_device( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mmproj_path = GgufSource::HuggingFace { + repo: "unsloth/Qwen3.5-0.8B-GGUF", + file: "mmproj-F16.gguf", + } + .resolve_path()?; + let model_path_c = CString::new(fixture.model_path.as_os_str().as_encoded_bytes())?; + let mmproj_path_c = CString::new(mmproj_path.as_os_str().as_encoded_bytes())?; + + let mut params = pin!(LlamaModelParams::default()); + let mut context_params = LlamaContextParams::default(); + let mut margins = vec![0usize; max_devices()]; + + MtmdMemoryUsage::measure(&mmproj_path_c, &MtmdContextParams::default())? + .add_to_fit_margins(&mut margins); + + assert_ne!(margins[0], 0, "{margins:?}"); + + let fit = params.as_mut().fit_params( + &model_path_c, + &mut context_params, + &mut margins, + 512, + None, + llama_cpp_bindings_sys::GGML_LOG_LEVEL_NONE, + )?; + + assert!(fit.n_ctx > 0); + + Ok(()) +} diff --git a/llama-cpp-bindings/src/mtmd.rs b/llama-cpp-bindings/src/mtmd.rs index 6db9270f0..38b20d450 100644 --- a/llama-cpp-bindings/src/mtmd.rs +++ b/llama-cpp-bindings/src/mtmd.rs @@ -7,11 +7,13 @@ pub mod mtmd_bitmap_error; pub mod mtmd_capability; pub mod mtmd_context; pub mod mtmd_context_params; +pub mod mtmd_context_params_error; pub mod mtmd_default_marker; pub mod mtmd_default_marker_error; pub mod mtmd_encode_error; pub mod mtmd_eval_error; pub mod mtmd_eval_limits; +pub mod mtmd_image_token_limit; pub mod mtmd_init_error; pub mod mtmd_input_chunk; pub mod mtmd_input_chunk_error; @@ -22,6 +24,8 @@ pub mod mtmd_input_chunks_error; pub mod mtmd_input_part; pub mod mtmd_input_text; pub mod mtmd_input_text_part; +pub mod mtmd_memory_usage; +pub mod mtmd_memory_usage_error; pub mod mtmd_tokenize_error; pub mod non_causal_chunk_micro_batch_mismatch; @@ -32,11 +36,13 @@ pub use mtmd_bitmap_error::MtmdBitmapError; pub use mtmd_capability::MtmdCapability; pub use mtmd_context::MtmdContext; pub use mtmd_context_params::MtmdContextParams; +pub use mtmd_context_params_error::MtmdContextParamsError; pub use mtmd_default_marker::mtmd_default_marker; pub use mtmd_default_marker_error::MtmdDefaultMarkerError; pub use mtmd_encode_error::MtmdEncodeError; pub use mtmd_eval_error::MtmdEvalError; pub use mtmd_eval_limits::MtmdEvalLimits; +pub use mtmd_image_token_limit::MtmdImageTokenLimit; pub use mtmd_init_error::MtmdInitError; pub use mtmd_input_chunk::MtmdInputChunk; pub use mtmd_input_chunk_error::MtmdInputChunkError; @@ -47,5 +53,7 @@ pub use mtmd_input_chunks_error::MtmdInputChunksError; pub use mtmd_input_part::MtmdInputPart; pub use mtmd_input_text::MtmdInputText; pub use mtmd_input_text_part::MtmdInputTextPart; +pub use mtmd_memory_usage::MtmdMemoryUsage; +pub use mtmd_memory_usage_error::MtmdMemoryUsageError; pub use mtmd_tokenize_error::MtmdTokenizeError; pub use non_causal_chunk_micro_batch_mismatch::NonCausalChunkMicroBatchMismatch; diff --git a/llama-cpp-bindings/src/mtmd/mtmd_context.rs b/llama-cpp-bindings/src/mtmd/mtmd_context.rs index 5d071405e..5b15faeee 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_context.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_context.rs @@ -318,7 +318,7 @@ impl MtmdContext { params: &MtmdContextParams, ) -> Result { let path_cstr = CString::new(mmproj_path)?; - let ctx_params = llama_cpp_bindings_sys::mtmd_context_params::from(params); + let ctx_params = llama_cpp_bindings_sys::mtmd_context_params::try_from(params)?; let mut out_ctx: *mut llama_cpp_bindings_sys::mtmd_context = ptr::null_mut(); let mut out_error: *mut c_char = ptr::null_mut(); diff --git a/llama-cpp-bindings/src/mtmd/mtmd_context_params.rs b/llama-cpp-bindings/src/mtmd/mtmd_context_params.rs index b850580bd..ebf5e630a 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_context_params.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_context_params.rs @@ -1,11 +1,16 @@ use std::ffi::{CStr, CString}; -#[derive(Debug, Clone)] +use super::mtmd_context_params_error::MtmdContextParamsError; +use super::mtmd_image_token_limit::MtmdImageTokenLimit; + +#[derive(Debug, Clone, PartialEq, Eq)] pub struct MtmdContextParams { pub use_gpu: bool, pub print_timings: bool, pub n_threads: i32, pub media_marker: CString, + pub image_min_tokens: MtmdImageTokenLimit, + pub image_max_tokens: MtmdImageTokenLimit, } impl Default for MtmdContextParams { @@ -14,22 +19,38 @@ impl Default for MtmdContextParams { } } -impl From<&MtmdContextParams> for llama_cpp_bindings_sys::mtmd_context_params { - fn from(params: &MtmdContextParams) -> Self { +impl TryFrom<&MtmdContextParams> for llama_cpp_bindings_sys::mtmd_context_params { + type Error = MtmdContextParamsError; + + fn try_from(params: &MtmdContextParams) -> Result { let mut context = unsafe { llama_cpp_bindings_sys::mtmd_context_params_default() }; let MtmdContextParams { use_gpu, print_timings, n_threads, media_marker, + image_min_tokens, + image_max_tokens, } = params; context.use_gpu = *use_gpu; context.print_timings = *print_timings; context.n_threads = *n_threads; context.media_marker = media_marker.as_ptr(); + context.image_min_tokens = image_min_tokens.to_c_int().map_err(|source| { + MtmdContextParamsError::ImageMinTokensExceedCInt { + limit: *image_min_tokens, + source, + } + })?; + context.image_max_tokens = image_max_tokens.to_c_int().map_err(|source| { + MtmdContextParamsError::ImageMaxTokensExceedCInt { + limit: *image_max_tokens, + source, + } + })?; - context + Ok(context) } } @@ -40,6 +61,72 @@ impl From for MtmdContextParams { print_timings: params.print_timings, n_threads: params.n_threads, media_marker: unsafe { CStr::from_ptr(params.media_marker) }.to_owned(), + image_min_tokens: MtmdImageTokenLimit::from_c_int(params.image_min_tokens), + image_max_tokens: MtmdImageTokenLimit::from_c_int(params.image_max_tokens), } } } + +#[cfg(test)] +mod tests { + use std::num::NonZeroU32; + + use super::MtmdContextParams; + use crate::mtmd::mtmd_context_params_error::MtmdContextParamsError; + use crate::mtmd::mtmd_image_token_limit::MtmdImageTokenLimit; + + const SIXTEEN: NonZeroU32 = NonZeroU32::new(16).unwrap(); + const SIXTY_FOUR: NonZeroU32 = NonZeroU32::new(64).unwrap(); + const UNREPRESENTABLE_LIMIT: MtmdImageTokenLimit = MtmdImageTokenLimit::Tokens(NonZeroU32::MAX); + + #[test] + fn image_token_limits_reach_the_c_parameters() { + let params = MtmdContextParams { + image_min_tokens: MtmdImageTokenLimit::Tokens(SIXTEEN), + image_max_tokens: MtmdImageTokenLimit::Tokens(SIXTY_FOUR), + ..MtmdContextParams::default() + }; + + assert_eq!( + llama_cpp_bindings_sys::mtmd_context_params::try_from(¶ms) + .map(|c_params| [c_params.image_min_tokens, c_params.image_max_tokens]), + Ok([16, 64]) + ); + } + + #[test] + fn an_unrepresentable_image_min_token_limit_is_refused() { + let params = MtmdContextParams { + image_min_tokens: UNREPRESENTABLE_LIMIT, + ..MtmdContextParams::default() + }; + + assert_eq!( + llama_cpp_bindings_sys::mtmd_context_params::try_from(¶ms).err(), + i32::try_from(u32::MAX).err().map(|source| { + MtmdContextParamsError::ImageMinTokensExceedCInt { + limit: UNREPRESENTABLE_LIMIT, + source, + } + }) + ); + } + + #[test] + fn an_unrepresentable_image_max_token_limit_is_refused() { + let params = MtmdContextParams { + image_max_tokens: UNREPRESENTABLE_LIMIT, + ..MtmdContextParams::default() + }; + + assert_eq!( + llama_cpp_bindings_sys::mtmd_context_params::try_from(¶ms).err(), + i32::try_from(u32::MAX).err().map(|source| { + MtmdContextParamsError::ImageMaxTokensExceedCInt { + limit: UNREPRESENTABLE_LIMIT, + source, + } + }) + ); + } +} diff --git a/llama-cpp-bindings/src/mtmd/mtmd_context_params_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_context_params_error.rs new file mode 100644 index 000000000..d4ad93886 --- /dev/null +++ b/llama-cpp-bindings/src/mtmd/mtmd_context_params_error.rs @@ -0,0 +1,19 @@ +use std::num::TryFromIntError; + +use super::mtmd_image_token_limit::MtmdImageTokenLimit; + +#[derive(thiserror::Error, Debug, PartialEq, Eq)] +pub enum MtmdContextParamsError { + #[error("the image minimum token limit {limit:?} exceeds what a C int holds")] + ImageMinTokensExceedCInt { + limit: MtmdImageTokenLimit, + #[source] + source: TryFromIntError, + }, + #[error("the image maximum token limit {limit:?} exceeds what a C int holds")] + ImageMaxTokensExceedCInt { + limit: MtmdImageTokenLimit, + #[source] + source: TryFromIntError, + }, +} diff --git a/llama-cpp-bindings/src/mtmd/mtmd_image_token_limit.rs b/llama-cpp-bindings/src/mtmd/mtmd_image_token_limit.rs new file mode 100644 index 000000000..c2a832ba7 --- /dev/null +++ b/llama-cpp-bindings/src/mtmd/mtmd_image_token_limit.rs @@ -0,0 +1,98 @@ +use std::ffi::c_int; +use std::num::NonZeroU32; +use std::num::TryFromIntError; + +/// A bound on the number of tokens the vision encoder produces for one image of a model with +/// dynamic resolution. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MtmdImageTokenLimit { + ModelMetadata, + Tokens(NonZeroU32), +} + +impl MtmdImageTokenLimit { + #[must_use] + pub fn from_c_int(raw: c_int) -> Self { + u32::try_from(raw) + .ok() + .and_then(NonZeroU32::new) + .map_or(Self::ModelMetadata, Self::Tokens) + } + + /// # Errors + /// + /// Returns the conversion error when the token count exceeds what a C `int` holds. + pub fn to_c_int(self) -> Result { + match self { + Self::ModelMetadata => Ok(-1), + Self::Tokens(tokens) => c_int::try_from(tokens.get()), + } + } + + #[must_use] + pub fn at_most(self, ceiling: NonZeroU32) -> Self { + match self { + Self::ModelMetadata => Self::ModelMetadata, + Self::Tokens(tokens) => Self::Tokens(tokens.min(ceiling)), + } + } +} + +#[cfg(test)] +mod tests { + use std::num::NonZeroU32; + + use super::MtmdImageTokenLimit; + + const SIXTY_FOUR: NonZeroU32 = NonZeroU32::new(64).unwrap(); + + #[test] + fn a_non_positive_c_value_reads_the_limit_from_the_model_metadata() { + assert_eq!( + [-1, 0].map(MtmdImageTokenLimit::from_c_int), + [MtmdImageTokenLimit::ModelMetadata; 2] + ); + } + + #[test] + fn a_positive_c_value_is_a_token_limit() { + assert_eq!( + MtmdImageTokenLimit::from_c_int(64), + MtmdImageTokenLimit::Tokens(SIXTY_FOUR) + ); + } + + #[test] + fn the_model_metadata_limit_is_the_c_value_minus_one() { + assert_eq!(MtmdImageTokenLimit::ModelMetadata.to_c_int(), Ok(-1)); + } + + #[test] + fn a_token_limit_is_its_c_value() { + assert_eq!(MtmdImageTokenLimit::Tokens(SIXTY_FOUR).to_c_int(), Ok(64)); + } + + #[test] + fn a_token_limit_beyond_a_c_int_does_not_convert() { + assert_eq!( + MtmdImageTokenLimit::Tokens(NonZeroU32::MAX).to_c_int(), + i32::try_from(u32::MAX) + ); + } + + #[test] + fn a_token_limit_above_the_ceiling_drops_to_the_ceiling() { + assert_eq!( + MtmdImageTokenLimit::Tokens(NonZeroU32::MAX).at_most(SIXTY_FOUR), + MtmdImageTokenLimit::Tokens(SIXTY_FOUR) + ); + } + + #[test] + fn the_model_metadata_limit_stays_under_a_ceiling() { + assert_eq!( + MtmdImageTokenLimit::ModelMetadata.at_most(SIXTY_FOUR), + MtmdImageTokenLimit::ModelMetadata + ); + } +} diff --git a/llama-cpp-bindings/src/mtmd/mtmd_init_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_init_error.rs index 6f9f45cf1..21ea9deaa 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_init_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_init_error.rs @@ -1,11 +1,15 @@ use std::path::PathBuf; +use super::mtmd_context_params_error::MtmdContextParamsError; + #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdInitError { #[error(transparent)] FfiStatus(#[from] crate::FfiStatusError), #[error(transparent)] FfiContract(#[from] crate::FfiContractError), + #[error(transparent)] + ContextParams(#[from] MtmdContextParamsError), #[error("Failed to create CString from mmproj path: {0}")] CStringError(#[from] std::ffi::NulError), #[error("Mmproj path is not valid UTF-8: {0:?}")] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs new file mode 100644 index 000000000..0fcb69dba --- /dev/null +++ b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs @@ -0,0 +1,424 @@ +use std::ffi::CStr; +use std::ffi::c_char; +use std::num::NonZeroU32; +use std::ptr; + +use llama_cpp_ffi_status::read_and_free_cpp_string; + +use super::mtmd_context_params::MtmdContextParams; +use super::mtmd_image_token_limit::MtmdImageTokenLimit; +use super::mtmd_memory_usage_error::MtmdMemoryUsageError; + +const OPERATION: &str = "llama_rs_mtmd_get_memory_usage"; + +fn get_memory_usage_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_mtmd_get_memory_usage_status, + out_error: *mut c_char, +) -> Result<(), MtmdMemoryUsageError> { + match status { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_OK => Ok(()), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_MMPROJ_PATH_ARG => { + Err(contract_violation("was given a null mmproj_path argument")) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_DEVICE_MEMORY_ARG => Err( + contract_violation("was given a null out_device_memory argument"), + ), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_DEVICE_MEMORY_COUNT_ARG => { + Err(contract_violation( + "was given a null out_device_memory_count argument", + )) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_IMAGE_MAX_TOKENS_ARG => { + Err(contract_violation( + "was given a null out_image_max_tokens argument", + )) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_USE_NON_CAUSAL_ARG => Err( + contract_violation("was given a null out_use_non_causal argument"), + ), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_DEVICE_MEMORY_CAPACITY_EXCEEDED => { + Err(contract_violation( + "measured memory on more devices than the backend registers", + )) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_ERROR_STRING_ALLOCATION_FAILED => { + Err(MtmdMemoryUsageError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_LLAMA_CPP_OUT_OF_MEMORY => { + Err(MtmdMemoryUsageError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + OPERATION, + "reported a thrown C++ exception without an error message", + ) + }?; + + Err(MtmdMemoryUsageError::Reported { message }) + } + other => Err(crate::FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), + } +} + +const fn contract_violation(detail: &'static str) -> MtmdMemoryUsageError { + MtmdMemoryUsageError::FfiContract(crate::FfiContractError { + operation: OPERATION, + detail, + }) +} + +/// The worst-case memory a multimodal projector takes on each backend device, with the image +/// budget it was measured at, as llama.cpp estimates it before the projector is loaded. +#[derive(Debug)] +pub struct MtmdMemoryUsage { + device_memory: Vec, + image_max_tokens: i32, + decodes_images_non_causally: bool, +} + +impl MtmdMemoryUsage { + /// Measures the memory the projector at `mmproj_path` takes when loaded with `params`. + /// + /// # Errors + /// + /// Returns [`MtmdMemoryUsageError::Unloadable`] when llama.cpp cannot load the projector, + /// [`MtmdMemoryUsageError::ContextParams`] when `params` have no C representation, or the + /// variant matching the wrapper's status code. + pub fn measure( + mmproj_path: &CStr, + params: &MtmdContextParams, + ) -> Result { + let context_params = llama_cpp_bindings_sys::mtmd_context_params::try_from(params)?; + let device_count = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_count() }; + let mut device_memory = Vec::with_capacity(device_count); + let mut device_memory_count: usize = 0; + let mut image_max_tokens: i32 = 0; + let mut decodes_images_non_causally = false; + let mut out_error: *mut c_char = ptr::null_mut(); + + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_mtmd_get_memory_usage( + mmproj_path.as_ptr(), + context_params, + device_memory.as_mut_ptr(), + device_count, + &raw mut device_memory_count, + &raw mut image_max_tokens, + &raw mut decodes_images_non_causally, + &raw mut out_error, + ) + }; + + get_memory_usage_status_to_result(status, out_error)?; + + unsafe { device_memory.set_len(device_memory_count) }; + + if device_memory.is_empty() { + return Err(MtmdMemoryUsageError::Unloadable { + path: mmproj_path.to_owned(), + }); + } + + Ok(Self { + device_memory, + image_max_tokens, + decodes_images_non_causally, + }) + } + + /// Adds the projector's memory to the fit margin of each device it takes memory on, so + /// fitting the text model leaves that memory free, as llama-server does. `margins` is + /// indexed by backend device, like the margins of + /// [`LlamaModelParams::fit_params`](crate::model::params::LlamaModelParams::fit_params); + /// a device past its end has no margin to grow. + pub fn add_to_fit_margins(&self, margins: &mut [usize]) { + let device_count = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_count() }; + + for llama_cpp_bindings_sys::llama_rs_mtmd_device_memory { device, bytes } in + &self.device_memory + { + let margin = (0..device_count) + .position(|device_index| { + let registered_device = + unsafe { llama_cpp_bindings_sys::ggml_backend_dev_get(device_index) }; + + registered_device == *device + }) + .and_then(|device_index| margins.get_mut(device_index)); + + if let Some(margin) = margin { + *margin += bytes; + } + } + } + + /// Lowers the image token limits of `params` to `micro_batch_tokens` when the projector + /// decodes images non-causally and could produce more tokens than one micro-batch holds, + /// because a non-causal image must be decoded in a single micro-batch. + #[must_use] + pub fn fit_image_tokens_into_micro_batch( + &self, + params: MtmdContextParams, + micro_batch_tokens: NonZeroU32, + ) -> MtmdContextParams { + if !self.decodes_images_non_causally + || i64::from(self.image_max_tokens) <= i64::from(micro_batch_tokens.get()) + { + return params; + } + + MtmdContextParams { + image_min_tokens: params.image_min_tokens.at_most(micro_batch_tokens), + image_max_tokens: MtmdImageTokenLimit::Tokens(micro_batch_tokens), + ..params + } + } +} + +#[cfg(test)] +mod tests { + use std::num::NonZeroU32; + use std::ptr; + + use super::MtmdMemoryUsage; + use super::get_memory_usage_status_to_result; + use crate::mtmd::mtmd_context_params::MtmdContextParams; + use crate::mtmd::mtmd_context_params_error::MtmdContextParamsError; + use crate::mtmd::mtmd_image_token_limit::MtmdImageTokenLimit; + use crate::mtmd::mtmd_memory_usage_error::MtmdMemoryUsageError; + + const MICRO_BATCH_TOKENS: NonZeroU32 = NonZeroU32::new(64).unwrap(); + + fn first_device_usage(bytes: usize) -> MtmdMemoryUsage { + MtmdMemoryUsage { + device_memory: vec![llama_cpp_bindings_sys::llama_rs_mtmd_device_memory { + device: unsafe { llama_cpp_bindings_sys::ggml_backend_dev_get(0) }, + bytes, + }], + image_max_tokens: -1, + decodes_images_non_causally: false, + } + } + + fn image_usage(image_max_tokens: i32, decodes_images_non_causally: bool) -> MtmdMemoryUsage { + MtmdMemoryUsage { + device_memory: Vec::new(), + image_max_tokens, + decodes_images_non_causally, + } + } + + fn params_with_min_tokens(image_min_tokens: MtmdImageTokenLimit) -> MtmdContextParams { + MtmdContextParams { + image_min_tokens, + ..MtmdContextParams::default() + } + } + + fn contract_violation(detail: &'static str) -> MtmdMemoryUsageError { + crate::FfiContractError { + operation: "llama_rs_mtmd_get_memory_usage", + detail, + } + .into() + } + + #[test] + fn measuring_a_missing_projector_reports_it_unloadable() { + assert_eq!( + MtmdMemoryUsage::measure(c"/nonexistent/mmproj.gguf", &MtmdContextParams::default()) + .err(), + Some(MtmdMemoryUsageError::Unloadable { + path: c"/nonexistent/mmproj.gguf".to_owned(), + }) + ); + } + + #[test] + fn measuring_with_unrepresentable_params_is_refused() { + let params = MtmdContextParams { + image_max_tokens: MtmdImageTokenLimit::Tokens(NonZeroU32::MAX), + ..MtmdContextParams::default() + }; + + assert_eq!( + MtmdMemoryUsage::measure(c"/nonexistent/mmproj.gguf", ¶ms).err(), + i32::try_from(u32::MAX).err().map(|source| { + MtmdMemoryUsageError::ContextParams( + MtmdContextParamsError::ImageMaxTokensExceedCInt { + limit: MtmdImageTokenLimit::Tokens(NonZeroU32::MAX), + source, + }, + ) + }) + ); + } + + #[test] + fn projector_memory_grows_the_margin_of_its_device() { + let mut margins = [1, 2]; + + first_device_usage(5).add_to_fit_margins(&mut margins); + + assert_eq!(margins, [6, 2]); + } + + #[test] + fn projector_memory_on_a_device_without_a_margin_is_not_added() { + let mut margins: [usize; 0] = []; + + first_device_usage(5).add_to_fit_margins(&mut margins); + + assert_eq!(margins, [0_usize; 0]); + } + + #[test] + fn a_causal_projector_keeps_the_image_token_limits() { + let params = params_with_min_tokens(MtmdImageTokenLimit::Tokens(NonZeroU32::MAX)); + + assert_eq!( + image_usage(4096, false) + .fit_image_tokens_into_micro_batch(params.clone(), MICRO_BATCH_TOKENS), + params + ); + } + + #[test] + fn a_non_causal_projector_within_the_micro_batch_keeps_the_image_token_limits() { + let params = params_with_min_tokens(MtmdImageTokenLimit::Tokens(NonZeroU32::MAX)); + + assert_eq!( + image_usage(64, true) + .fit_image_tokens_into_micro_batch(params.clone(), MICRO_BATCH_TOKENS), + params + ); + } + + #[test] + fn a_non_causal_projector_beyond_the_micro_batch_lowers_the_image_token_limits_to_it() { + let params = params_with_min_tokens(MtmdImageTokenLimit::Tokens(NonZeroU32::MAX)); + + assert_eq!( + image_usage(65, true) + .fit_image_tokens_into_micro_batch(params.clone(), MICRO_BATCH_TOKENS), + MtmdContextParams { + image_min_tokens: MtmdImageTokenLimit::Tokens(MICRO_BATCH_TOKENS), + image_max_tokens: MtmdImageTokenLimit::Tokens(MICRO_BATCH_TOKENS), + ..params + } + ); + } + + #[test] + fn the_ok_status_is_success() { + assert_eq!( + get_memory_usage_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_OK, + ptr::null_mut(), + ), + Ok(()) + ); + } + + #[test] + fn wrapper_contract_statuses_are_contract_violations() { + let statuses = [ + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_MMPROJ_PATH_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_DEVICE_MEMORY_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_DEVICE_MEMORY_COUNT_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_IMAGE_MAX_TOKENS_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_USE_NON_CAUSAL_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_DEVICE_MEMORY_CAPACITY_EXCEEDED, + ]; + + assert_eq!( + statuses.map(|status| get_memory_usage_status_to_result(status, ptr::null_mut())), + [ + Err(contract_violation("was given a null mmproj_path argument")), + Err(contract_violation( + "was given a null out_device_memory argument" + )), + Err(contract_violation( + "was given a null out_device_memory_count argument" + )), + Err(contract_violation( + "was given a null out_image_max_tokens argument" + )), + Err(contract_violation( + "was given a null out_use_non_causal argument" + )), + Err(contract_violation( + "measured memory on more devices than the backend registers" + )), + ] + ); + } + + #[test] + fn a_failed_error_string_allocation_is_not_enough_memory() { + assert_eq!( + get_memory_usage_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_ERROR_STRING_ALLOCATION_FAILED, + ptr::null_mut(), + ), + Err(MtmdMemoryUsageError::NotEnoughMemory) + ); + } + + #[test] + fn llama_cpp_running_out_of_memory_is_reported() { + assert_eq!( + get_memory_usage_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_LLAMA_CPP_OUT_OF_MEMORY, + ptr::null_mut(), + ), + Err(MtmdMemoryUsageError::LlamaCppOutOfMemory) + ); + } + + #[test] + fn a_thrown_exception_reports_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"projector threw".as_ptr()) }; + + assert_eq!( + get_memory_usage_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_LLAMA_CPP_THREW_CXX_EXCEPTION, + out_error, + ), + Err(MtmdMemoryUsageError::Reported { + message: "projector threw".to_owned(), + }) + ); + } + + #[test] + fn a_thrown_exception_without_a_message_is_a_contract_violation() { + assert_eq!( + get_memory_usage_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + ), + Err(contract_violation( + "reported a thrown C++ exception without an error message" + )) + ); + } + + #[test] + fn an_unknown_status_is_preserved() { + assert_eq!( + get_memory_usage_status_to_result(255, ptr::null_mut()), + Err(MtmdMemoryUsageError::FfiStatus(crate::FfiStatusError { + operation: "llama_rs_mtmd_get_memory_usage", + code: 255, + })) + ); + } +} diff --git a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage_error.rs new file mode 100644 index 000000000..ac339cafc --- /dev/null +++ b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage_error.rs @@ -0,0 +1,21 @@ +use std::ffi::CString; + +use super::mtmd_context_params_error::MtmdContextParamsError; + +#[derive(thiserror::Error, Debug, PartialEq, Eq)] +pub enum MtmdMemoryUsageError { + #[error(transparent)] + FfiStatus(#[from] crate::FfiStatusError), + #[error(transparent)] + FfiContract(#[from] crate::FfiContractError), + #[error(transparent)] + ContextParams(#[from] MtmdContextParamsError), + #[error("the memory usage of mmproj {path:?} could not be measured, because it does not load")] + Unloadable { path: CString }, + #[error("not enough memory")] + NotEnoughMemory, + #[error("the llama.cpp library ran out of memory")] + LlamaCppOutOfMemory, + #[error("{message}")] + Reported { message: String }, +} From b56bd76e5bbaa7326f3053f654a7944ece3b1f6f Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Fri, 30 Oct 2026 14:11:29 +0100 Subject: [PATCH 09/82] Validate token buffer sizes before allocating, refuse NUL paths before probing them, and test the error paths of sampling, logits, chat sessions and duck typing --- .../tests/chat_session_input_validation.rs | 107 ++++++++++++++++++ .../tests/chat_template_lookup.rs | 19 ++++ .../tests/detokenization.rs | 24 ++++ .../tests/duck_typed_tool_calls.rs | 30 +++++ .../tests/grammar_sampler.rs | 19 ++++ .../tests/llguidance_sampler.rs | 65 +++++++++-- .../tests/logits_availability.rs | 66 +++++++++++ .../tests/model_loading_errors.rs | 43 +++++++ .../tests/prompt_token_accounting.rs | 50 ++++++++ .../src/bare_json_tool_call_probe.rs | 14 +++ llama-cpp-bindings/src/chat_session.rs | 62 ++++++++-- llama-cpp-bindings/src/gguf_context.rs | 3 +- llama-cpp-bindings/src/grammar_matcher.rs | 42 +++++++ llama-cpp-bindings/src/llama_batch.rs | 16 +++ llama-cpp-bindings/src/llguidance_sampler.rs | 54 +++++---- llama-cpp-bindings/src/model.rs | 77 +++++++++---- llama-cpp-bindings/src/model/params.rs | 37 ++---- llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs | 9 +- .../src/mtmd/mtmd_default_marker.rs | 13 ++- llama-cpp-bindings/src/sampling.rs | 19 ++++ llama-cpp-bindings/src/send_logs_to_log.rs | 5 +- .../src/streaming_json_probe.rs | 5 + llama-cpp-bindings/src/token/data_array.rs | 88 ++++++++++++++ 23 files changed, 766 insertions(+), 101 deletions(-) diff --git a/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs index f83e3e975..20113e3dc 100644 --- a/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs +++ b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs @@ -1,7 +1,15 @@ +use std::ffi::CString; + use anyhow::Result; use anyhow::bail; +use llama_cpp_bindings::ChatAssistantTurn; +use llama_cpp_bindings::ChatSession; use llama_cpp_bindings::ChatSessionError; +use llama_cpp_bindings::ChatSessionOptions; +use llama_cpp_bindings::ChatToolChoice; use llama_cpp_bindings::NewLlamaChatMessageError; +use llama_cpp_bindings::StringToTokenError; +use llama_cpp_bindings::model::LlamaChatMessage; use llama_cpp_bindings_tests::media_question_chat_session::media_question_chat_session; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; use llama_cpp_test_harness::LlamaFixture; @@ -56,3 +64,102 @@ fn chat_session_refuses_a_tool_missing_its_function_name(fixture: &LlamaFixture< Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn chat_session_refuses_tools_with_a_nul_byte(fixture: &LlamaFixture<'_>) { + assert_eq!( + user_turn_chat_session(fixture.model, "hello", "[\0]", false) + .err() + .and_then(|failure| failure.downcast::().ok()), + CString::new("[\0]") + .err() + .map(ChatSessionError::RequestContainsNul) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn chat_session_refuses_a_message_that_is_not_utf8(fixture: &LlamaFixture<'_>) -> Result<()> { + let not_utf8 = CString::new([0xff])?; + let message = LlamaChatMessage { + role: CString::new("user")?, + content: not_utf8.clone(), + }; + + assert_eq!( + ChatSession::new( + fixture.model, + &[message], + "[]", + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::New, + enable_thinking: false, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + ) + .err(), + not_utf8 + .to_str() + .err() + .map(ChatSessionError::MessageNotUtf8) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn tokenizing_a_prompt_whose_reasoning_prefill_holds_a_nul_character_is_refused( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = ChatSession::new( + fixture.model, + &[LlamaChatMessage::new( + "user".to_owned(), + "hello".to_owned(), + )?], + "[]", + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::ContinueReasoning("a\0b".to_owned()), + enable_thinking: true, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + )?; + let Some(nul_error) = session + .prompt_parts()? + .iter() + .find_map(|part| CString::new(part.text()).err()) + else { + bail!("the rendered prompt must hold the reasoning prefill's NUL character"); + }; + + assert_eq!( + session.tokenize_prompt().err(), + Some(ChatSessionError::PromptTokenization( + StringToTokenError::NulError(nul_error) + )) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/chat_template_lookup.rs b/llama-cpp-bindings-tests/tests/chat_template_lookup.rs index 1215b8018..7102335a6 100644 --- a/llama-cpp-bindings-tests/tests/chat_template_lookup.rs +++ b/llama-cpp-bindings-tests/tests/chat_template_lookup.rs @@ -1,3 +1,5 @@ +use std::ffi::CString; + use llama_cpp_bindings::ChatTemplateError; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -77,3 +79,20 @@ fn chat_template_with_nonexistent_name_returns_error(fixture: &LlamaFixture<'_>) .chat_template(Some("nonexistent_template_name_xyz")); assert_eq!(result.unwrap_err(), ChatTemplateError::MissingTemplate); } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn chat_template_with_a_nul_byte_in_the_name_returns_error(fixture: &LlamaFixture<'_>) { + assert_eq!( + fixture.model.chat_template(Some("tool\0use")).err(), + CString::new("tool\0use") + .err() + .map(ChatTemplateError::NullError) + ); +} diff --git a/llama-cpp-bindings-tests/tests/detokenization.rs b/llama-cpp-bindings-tests/tests/detokenization.rs index e7d1597e0..9751b2d2d 100644 --- a/llama-cpp-bindings-tests/tests/detokenization.rs +++ b/llama-cpp-bindings-tests/tests/detokenization.rs @@ -2,8 +2,10 @@ use std::num::NonZeroU16; use anyhow::Result; use llama_cpp_bindings::SampledToken; +use llama_cpp_bindings::TokenToStringError; use llama_cpp_bindings::model::AddBos; use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::token::LlamaToken; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -324,3 +326,25 @@ fn token_to_piece_decodes_undeterminable_variant(fixture: &LlamaFixture<'_>) -> assert_ne!(piece, ""); Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_to_piece_bytes_refuses_a_buffer_beyond_a_c_int_before_allocating( + fixture: &LlamaFixture<'_>, +) { + assert_eq!( + fixture + .model + .token_to_piece_bytes(LlamaToken::new(0), usize::MAX, false, None) + .err(), + i32::try_from(usize::MAX) + .err() + .map(TokenToStringError::IntConversionError) + ); +} diff --git a/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs b/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs index 19376373a..674605687 100644 --- a/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs +++ b/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs @@ -83,3 +83,33 @@ fn deepseek_r1_8b_duck_types_qwen_xml(fixture: &LlamaFixture<'_>) -> Result<()> "\n\n\nParis\n\n\n", ) } + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn deepseek_r1_8b_duck_types_qwen_json_inside_tool_call(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_duck_types_paris_weather_call( + fixture, + "\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Paris\"}}\n", + ) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn deepseek_r1_8b_duck_types_a_bare_json_tool_call(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_duck_types_paris_weather_call( + fixture, + "{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Paris\"}}", + ) +} diff --git a/llama-cpp-bindings-tests/tests/grammar_sampler.rs b/llama-cpp-bindings-tests/tests/grammar_sampler.rs index 279f3900a..89bf0b90f 100644 --- a/llama-cpp-bindings-tests/tests/grammar_sampler.rs +++ b/llama-cpp-bindings-tests/tests/grammar_sampler.rs @@ -1,3 +1,5 @@ +use std::ffi::CString; + use anyhow::Result; use anyhow::anyhow; use anyhow::bail; @@ -291,3 +293,20 @@ fn qwen35_grammar_with_a_syntax_error_reports_it(fixture: &LlamaFixture<'_>) { )) ); } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn a_grammar_with_a_nul_byte_is_refused(fixture: &LlamaFixture<'_>) { + assert_eq!( + LlamaSampler::grammar(fixture.model, "root ::= \"a\0\"", "root").err(), + CString::new("root ::= \"a\0\"") + .err() + .map(GrammarError::GrammarContainsNul) + ); +} diff --git a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs index f57806cb5..e2b654ae2 100644 --- a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs +++ b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs @@ -3,9 +3,11 @@ use std::ffi::CStr; use anyhow::Context as _; use anyhow::Result; use anyhow::anyhow; +use llama_cpp_bindings::SampleError; use llama_cpp_bindings::SamplerAcceptError; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::error::GrammarRuntimeError; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::llguidance_sampler::create_llg_sampler; use llama_cpp_bindings::model::AddBos; @@ -18,6 +20,13 @@ use llama_cpp_test_harness::llama_test; const JSON_SCHEMA: &str = r#"{"type":"object","properties":{"answer":{"type":"string"}},"required":["answer"]}"#; const REGEX_GRAMMAR: &str = r"yes|no"; + +fn out_of_vocabulary_failure() -> String { + GrammarRuntimeError::InternalParserError { + message: "token id 2147483646 out of range".to_owned(), + } + .to_string() +} const LARK_GRAMMAR: &str = r#"start: "yes" | "no""#; #[llama_test( @@ -488,13 +497,12 @@ fn accept_maps_an_out_of_vocabulary_token_to_grammar_callback_failure( ) -> Result<()> { let mut sampler = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; - let huge_token = LlamaToken(i32::MAX - 1); - let result = sampler.accept(huge_token); - - let Err(SamplerAcceptError::GrammarCallbackFailed { message }) = result else { - panic!("expected a grammar callback failure, got {result:?}"); - }; - assert!(message.contains("token id 2147483646 out of range")); + assert_eq!( + sampler.accept(LlamaToken(i32::MAX - 1)), + Err(SamplerAcceptError::GrammarCallbackFailed { + message: out_of_vocabulary_failure(), + }) + ); Ok(()) } @@ -597,3 +605,46 @@ fn llguidance_chain_samples_a_valid_token(fixture: &LlamaFixture<'_>) -> Result< Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn sampling_through_a_grammar_in_an_error_state_reports_the_grammar_failure( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let mut context = fixture.build_context()?; + let tokens = model.str_to_token( + "Answer yes or no:", + AddBos::Always, + ParseSpecialTokens::Always, + )?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let mut chain = LlamaSampler::chain_simple([ + create_llg_sampler(model, "regex", REGEX_GRAMMAR)?, + LlamaSampler::greedy()?, + ])?; + + assert_eq!( + chain.accept(LlamaToken(i32::MAX - 1)), + Err(SamplerAcceptError::GrammarCallbackFailed { + message: out_of_vocabulary_failure(), + }) + ); + assert_eq!( + chain.sample(&context, batch.n_tokens() - 1), + Err(SampleError::GrammarCallbackFailed { + message: out_of_vocabulary_failure(), + }) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/logits_availability.rs b/llama-cpp-bindings-tests/tests/logits_availability.rs index 76cc37c44..b2e399c48 100644 --- a/llama-cpp-bindings-tests/tests/logits_availability.rs +++ b/llama-cpp-bindings-tests/tests/logits_availability.rs @@ -1,8 +1,10 @@ use anyhow::Result; use llama_cpp_bindings::LogitsError; use llama_cpp_bindings::SampleError; +use llama_cpp_bindings::SampledTokenClassifier; use llama_cpp_bindings::context::params::LlamaPoolingType; use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings::token::data_array::LlamaTokenDataArray; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -78,3 +80,67 @@ fn a_model_whose_embeddings_graph_has_no_logits_head_produces_no_logits( Ok(()) } + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, + embeddings = true, +)] +fn every_logits_accessor_refuses_a_pooled_embeddings_decode( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean)?; + + assert_eq!( + [ + context.get_logits().err(), + context.candidates().err(), + context.token_data_array().err(), + context.candidates_ith(-1).err(), + context.token_data_array_ith(-1).err(), + context + .fill_token_data_array_ith(-1, &mut LlamaTokenDataArray::new(Vec::new(), false)) + .err(), + ], + [(); 6].map(|()| { + Some(LogitsError::PooledEmbeddingsProduceNoLogits { + pooling_type: LlamaPoolingType::Mean, + }) + }) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, + embeddings = true, +)] +fn classified_sampling_after_a_pooled_embeddings_decode_is_refused( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean)?; + let mut classifier = SampledTokenClassifier::undeterminable(fixture.model); + + assert_eq!( + classifier + .sample(&mut LlamaSampler::greedy()?, &context, -1, &mut Vec::new()) + .err(), + Some(SampleError::Logits( + LogitsError::PooledEmbeddingsProduceNoLogits { + pooling_type: LlamaPoolingType::Mean, + } + )) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/model_loading_errors.rs b/llama-cpp-bindings-tests/tests/model_loading_errors.rs index d6ab6b8db..134dfee24 100644 --- a/llama-cpp-bindings-tests/tests/model_loading_errors.rs +++ b/llama-cpp-bindings-tests/tests/model_loading_errors.rs @@ -1,3 +1,4 @@ +use std::ffi::CString; use std::path::Path; use std::path::PathBuf; @@ -135,3 +136,45 @@ fn lora_adapter_init_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_> LlamaLoraAdapterInitError::PathToStrError(non_utf8_path.to_path_buf()) ); } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn load_model_with_a_nul_byte_in_the_path_returns_error(fixture: &LlamaFixture<'_>) { + assert_eq!( + LlamaModel::load_from_file( + fixture.backend, + "/models/model\0.gguf", + &LlamaModelParams::default() + ) + .err(), + CString::new("/models/model\0.gguf") + .err() + .map(LlamaModelLoadError::NullError) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn lora_adapter_init_with_a_nul_byte_in_the_path_returns_error(fixture: &LlamaFixture<'_>) { + assert_eq!( + fixture + .model + .lora_adapter_init("/adapters/adapter\0.gguf") + .err(), + CString::new("/adapters/adapter\0.gguf") + .err() + .map(LlamaLoraAdapterInitError::NullError) + ); +} diff --git a/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs b/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs index 8b6c572f3..3f1216a57 100644 --- a/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs +++ b/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs @@ -2,8 +2,10 @@ use anyhow::Context as _; use anyhow::Result; use llama_cpp_bindings::SampledTokenClassifier; use llama_cpp_bindings::TokenUsageError; +use llama_cpp_bindings::batch_add_error::BatchAddError; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::token::LlamaToken; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -275,3 +277,51 @@ fn cached_prompt_tokens_cannot_exceed_the_recorded_prompt(fixture: &LlamaFixture }) ); } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn a_prompt_token_refused_by_a_full_batch_is_not_counted(fixture: &LlamaFixture<'_>) { + let mut classifier = SampledTokenClassifier::undeterminable(fixture.model); + let mut batch = LlamaBatch::new(ContextLimits { + n_batch: 0, + n_ubatch: 0, + n_seq_max: 1, + n_vocab: fixture.model.n_vocab(), + }); + + assert_eq!( + classifier.feed_prompt_to_batch(&mut batch, LlamaToken::new(0), 0, &[0], false), + Err(BatchAddError::BatchFull { n_batch: 0 }) + ); + assert_eq!(classifier.pending_prompt_tokens(), 0); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn a_prompt_sequence_refused_by_a_full_batch_is_not_counted(fixture: &LlamaFixture<'_>) { + let mut classifier = SampledTokenClassifier::undeterminable(fixture.model); + let mut batch = LlamaBatch::new(ContextLimits { + n_batch: 0, + n_ubatch: 0, + n_seq_max: 1, + n_vocab: fixture.model.n_vocab(), + }); + + assert_eq!( + classifier.feed_prompt_sequence_to_batch(&mut batch, &[LlamaToken::new(0)], 0, false), + Err(BatchAddError::BatchFull { n_batch: 0 }) + ); + assert_eq!(classifier.pending_prompt_tokens(), 0); +} diff --git a/llama-cpp-bindings/src/bare_json_tool_call_probe.rs b/llama-cpp-bindings/src/bare_json_tool_call_probe.rs index ad75957f8..8bb871b57 100644 --- a/llama-cpp-bindings/src/bare_json_tool_call_probe.rs +++ b/llama-cpp-bindings/src/bare_json_tool_call_probe.rs @@ -193,6 +193,20 @@ mod tests { ); } + #[test] + fn a_content_marker_releases_the_held_object_before_itself() { + let mut probe = BareJsonToolCallProbe::default(); + let marker = IngestOutcome { + sampled_token: SampledToken::Content(LlamaToken::new(3)), + piece: TokenPiece::Marker("<|im_end|>".to_owned()), + }; + + assert_eq!( + pass_all(&mut probe, vec![content("{\"name\":"), marker.clone()]), + [content("{\"name\":"), marker] + ); + } + #[test] fn two_consecutive_tool_call_objects_are_recognized_separately() { let mut probe = BareJsonToolCallProbe::default(); diff --git a/llama-cpp-bindings/src/chat_session.rs b/llama-cpp-bindings/src/chat_session.rs index b0475108b..90fa56902 100644 --- a/llama-cpp-bindings/src/chat_session.rs +++ b/llama-cpp-bindings/src/chat_session.rs @@ -760,6 +760,7 @@ impl Drop for ChatSession<'_> { #[cfg(test)] mod tests { use std::ffi::CStr; + use std::ffi::CString; use std::ptr; use serde_json::json; @@ -873,17 +874,25 @@ mod tests { #[test] fn creating_a_session_reports_a_refusal_without_a_message_as_a_contract_violation() { + let refusals = [ + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_MESSAGES_NOT_JSON, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_MESSAGES_INVALID, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_NOT_JSON, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_INVALID, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TEMPLATE_CANNOT_SERVE_REQUEST, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, + ]; + assert_eq!( - chat_session_create_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_INVALID, - ptr::null_mut(), - ptr::null_mut() - ) - .err(), - Some(ChatSessionError::FfiContract(crate::FfiContractError { - operation: "llama_rs_chat_session_create", - detail: "reported a failure without an error message" - })) + refusals.map(|status| { + chat_session_create_status_to_result(status, ptr::null_mut(), ptr::null_mut()).err() + }), + refusals.map(|_status| { + Some(ChatSessionError::FfiContract(crate::FfiContractError { + operation: "llama_rs_chat_session_create", + detail: "reported a failure without an error message", + })) + }) ); } @@ -1121,6 +1130,27 @@ mod tests { ); } + #[test] + fn a_message_whose_role_or_content_is_not_utf8_is_refused() { + let not_utf8 = CString::new([0xff]).unwrap(); + let utf8_error = not_utf8.to_str().err(); + let messages = [ + LlamaChatMessage { + role: not_utf8.clone(), + content: c"hi".to_owned(), + }, + LlamaChatMessage { + role: c"user".to_owned(), + content: not_utf8, + }, + ]; + + assert_eq!( + messages.map(|message| messages_json(&[message], &ChatAssistantTurn::New).err()), + [utf8_error, utf8_error].map(|error| error.map(ChatSessionError::MessageNotUtf8)) + ); + } + #[test] fn continuing_reasoning_appends_an_assistant_message_with_that_reasoning() { assert_eq!( @@ -1155,6 +1185,14 @@ mod tests { assert_eq!( [ + chat_session_parse_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_OUTPUT_DOES_NOT_MATCH_FORMAT, + ptr::null_mut(), + ptr::null_mut(), + &output, + ChatOutputCompleteness::Finished, + ) + .err(), chat_session_parse_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), @@ -1172,6 +1210,10 @@ mod tests { .err(), ], [ + Some(ParseChatMessageError::FfiContract(crate::FfiContractError { + operation: "llama_rs_chat_session_parse", + detail: "reported a failure without an error message", + })), Some(ParseChatMessageError::FfiContract(crate::FfiContractError { operation: "llama_rs_chat_session_parse", detail: "reported a failure without an error message", diff --git a/llama-cpp-bindings/src/gguf_context.rs b/llama-cpp-bindings/src/gguf_context.rs index 32ec8cd72..a96601920 100644 --- a/llama-cpp-bindings/src/gguf_context.rs +++ b/llama-cpp-bindings/src/gguf_context.rs @@ -620,8 +620,7 @@ mod tests { impl Drop for SyntheticGgufFile { fn drop(&mut self) { - std::fs::remove_file(&self.path) - .unwrap_or_else(|error| panic!("failed to remove synthetic GGUF: {error}")); + std::fs::remove_file(&self.path).unwrap(); } } diff --git a/llama-cpp-bindings/src/grammar_matcher.rs b/llama-cpp-bindings/src/grammar_matcher.rs index f7a61e5a5..cd512c13f 100644 --- a/llama-cpp-bindings/src/grammar_matcher.rs +++ b/llama-cpp-bindings/src/grammar_matcher.rs @@ -131,13 +131,55 @@ impl GrammarMatcher { #[cfg(test)] mod tests { + use llguidance::ParserFactory; + use llguidance::TokenParser; use llguidance::api::StopReason; + use llguidance::api::TopLevelGrammar; + use toktrie::ApproximateTokEnv; + use super::GrammarMatcher; use super::catch_step_panic; use super::panic_payload_message; use super::stop_reason_to_result; use crate::error::grammar_runtime_error::GrammarRuntimeError; + fn letters_a_then_b_parser() -> TokenParser { + ParserFactory::new_simple(&ApproximateTokEnv::single_byte_env()) + .unwrap() + .create_parser(TopLevelGrammar::from_regex("ab")) + .unwrap() + } + + #[test] + fn a_started_parser_keeps_the_tokens_it_consumed() { + let mut parser = letters_a_then_b_parser(); + + parser.start_without_prompt(); + parser.consume_token(u32::from(b'a')).unwrap(); + + assert_eq!( + GrammarMatcher::new(parser).consume_token(u32::from(b'b')), + Ok(()) + ); + } + + #[test] + fn a_panicking_parser_step_is_reported_with_its_operation() { + let mut matcher = GrammarMatcher::new(letters_a_then_b_parser()); + + assert_eq!( + matcher + .run("compute_mask", |_parser| -> Result<(), String> { + panic!("parser exploded") + }) + .err(), + Some(GrammarRuntimeError::Panicked { + operation: "compute_mask", + message: "parser exploded".to_owned(), + }) + ); + } + #[test] fn a_static_str_panic_payload_is_preserved() { let payload: Box = Box::new("parser exploded"); diff --git a/llama-cpp-bindings/src/llama_batch.rs b/llama-cpp-bindings/src/llama_batch.rs index bcebdb54d..f7930e4cd 100644 --- a/llama-cpp-bindings/src/llama_batch.rs +++ b/llama-cpp-bindings/src/llama_batch.rs @@ -150,6 +150,22 @@ mod tests { SampledToken::Content(LlamaToken::new(token_id)) } + #[test] + fn a_batch_whose_token_count_reached_its_type_limit_refuses_another_token() { + let mut batch = LlamaBatch { + n_tokens: i32::MAX, + ..LlamaBatch::new(ContextLimits { + n_batch: u32::MAX, + ..LIMITS + }) + }; + + assert_eq!( + batch.add(&content(1), 0, &[0], false), + Err(BatchAddError::TokenCountOverflow { n_tokens: i32::MAX }) + ); + } + #[test] fn added_tokens_are_kept_in_order_with_their_outputs() { let mut batch = LlamaBatch::new(LIMITS); diff --git a/llama-cpp-bindings/src/llguidance_sampler.rs b/llama-cpp-bindings/src/llguidance_sampler.rs index 07090280c..a55228044 100644 --- a/llama-cpp-bindings/src/llguidance_sampler.rs +++ b/llama-cpp-bindings/src/llguidance_sampler.rs @@ -87,10 +87,7 @@ unsafe extern "C" fn llg_clone( } unsafe extern "C" fn llg_free(smpl: *mut llama_cpp_bindings_sys::llama_sampler) { - let ctx_ptr = unsafe { (*smpl).ctx.cast::() }; - if !ctx_ptr.is_null() { - drop(unsafe { Box::from_raw(ctx_ptr) }); - } + drop(unsafe { Box::from_raw((*smpl).ctx.cast::()) }); } static mut LLG_SAMPLER_I: llama_cpp_bindings_sys::llama_sampler_i = @@ -169,7 +166,6 @@ pub fn create_llg_sampler( mod tests { use std::sync::Arc; - use anyhow::Result; use toktrie::ApproximateTokEnv; use toktrie::INVALID_TOKEN; use toktrie::TokRxInfo; @@ -217,21 +213,18 @@ mod tests { } #[test] - fn apply_leaves_candidates_untouched_once_the_grammar_is_complete() -> Result<()> { - let mut sampler = letter_a_sampler()?; + fn apply_leaves_candidates_untouched_once_the_grammar_is_complete() { + let mut sampler = letter_a_sampler().unwrap(); let mut candidates = letter_candidates(); - sampler.accept(LETTER_A)?; - - sampler.apply(&mut candidates)?; + assert_eq!(sampler.accept(LETTER_A), Ok(())); + assert_eq!(sampler.apply(&mut candidates), Ok(())); assert_eq!(candidates, letter_candidates()); - - Ok(()) } #[test] - fn apply_reports_a_grammar_in_an_error_state_as_a_callback_failure() -> Result<()> { - let mut sampler = letter_a_sampler()?; + fn apply_reports_a_grammar_in_an_error_state_as_a_callback_failure() { + let mut sampler = letter_a_sampler().unwrap(); assert_eq!( sampler.accept(OUT_OF_VOCABULARY), @@ -245,15 +238,38 @@ mod tests { message: out_of_vocabulary_failure(), }) ); + } + + #[test] + fn accepting_many_tokens_stops_at_the_first_one_the_grammar_rejects() { + let mut sampler = letter_a_sampler().unwrap(); - Ok(()) + assert_eq!( + sampler.accept_many([OUT_OF_VOCABULARY, LETTER_A]), + Err(SamplerAcceptError::GrammarCallbackFailed { + message: out_of_vocabulary_failure(), + }) + ); } #[test] - fn reset_reports_a_grammar_in_an_error_state_as_a_callback_failure() -> Result<()> { - let mut sampler = letter_a_sampler()?; - sampler.accept(LETTER_A)?; + fn a_sampler_with_a_rejected_token_is_not_built() { + assert_eq!( + letter_a_sampler() + .unwrap() + .with_tokens([OUT_OF_VOCABULARY]) + .err(), + Some(SamplerAcceptError::GrammarCallbackFailed { + message: out_of_vocabulary_failure(), + }) + ); + } + + #[test] + fn reset_reports_a_grammar_in_an_error_state_as_a_callback_failure() { + let mut sampler = letter_a_sampler().unwrap(); + assert_eq!(sampler.accept(LETTER_A), Ok(())); assert_eq!( sampler.accept(OUT_OF_VOCABULARY), Err(SamplerAcceptError::GrammarCallbackFailed { @@ -266,7 +282,5 @@ mod tests { message: out_of_vocabulary_failure(), }) ); - - Ok(()) } } diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index 9eb05c1c5..ddcd76a60 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -58,6 +58,8 @@ pub use vocab_type_from_int_error::VocabTypeFromIntError; use llama_cpp_ffi_status::read_and_free_cpp_string; use params::LlamaModelParams; +const PIECE_PROBE_SIZE: usize = 32; + fn validate_string_length_for_tokenizer(length: usize) -> Result { Ok(c_int::try_from(length)?) } @@ -330,14 +332,7 @@ impl LlamaModel { | SampledToken::Reasoning(inner) | SampledToken::ToolCall(inner) | SampledToken::Undeterminable(inner)) = *token; - let bytes = match self.token_to_piece_bytes(inner, 8, special, lstrip) { - Err(TokenToStringError::InsufficientBufferSpace(required_size)) => { - let buffer_size: usize = (-required_size).try_into()?; - - self.token_to_piece_bytes(inner, buffer_size, special, lstrip) - } - other => other, - }?; + let bytes = self.token_to_piece_bytes_fitted(inner, special, lstrip)?; let mut output_piece = String::with_capacity(bytes.len()); let (_result, _decoded_size, _had_replacements) = @@ -360,8 +355,8 @@ impl LlamaModel { lstrip: Option, ) -> Result, TokenToStringError> { let token = self.require_vocabulary_token(token)?; + let buffer_len = c_int::try_from(buffer_size)?; let mut buffer: Vec = vec![0u8; buffer_size]; - let buffer_len = c_int::try_from(buffer.len())?; let lstrip = lstrip.map_or(0, |strip_count| i32::from(strip_count.get())); let size = unsafe { llama_cpp_bindings_sys::llama_token_to_piece( @@ -380,14 +375,31 @@ impl LlamaModel { Err(TokenToStringError::InsufficientBufferSpace(error_code)) } size => { - let written = usize::try_from(size)?; - buffer.truncate(written); + buffer.truncate(size.unsigned_abs() as usize); Ok(buffer) } } } + fn token_to_piece_bytes_fitted( + &self, + token: LlamaToken, + special: bool, + lstrip: Option, + ) -> Result, TokenToStringError> { + match self.token_to_piece_bytes(token, PIECE_PROBE_SIZE, special, lstrip) { + Err(TokenToStringError::InsufficientBufferSpace(required_size)) => self + .token_to_piece_bytes( + token, + required_size.unsigned_abs() as usize, + special, + lstrip, + ), + probe_result => probe_result, + } + } + #[must_use] pub fn n_vocab(&self) -> i32 { unsafe { llama_cpp_bindings_sys::llama_vocab_n_tokens(self.vocab_ptr()) } @@ -561,12 +573,11 @@ impl LlamaModel { let path_str = path .to_str() .ok_or_else(|| LlamaModelLoadError::PathToStrError(path.to_path_buf()))?; + let cstr = CString::new(path_str)?; if !path.exists() { return Err(LlamaModelLoadError::FileNotFound(path.to_path_buf())); } - - let cstr = CString::new(path_str)?; let mut out_model: *mut llama_cpp_bindings_sys::llama_model = ptr::null_mut(); let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { @@ -592,12 +603,11 @@ impl LlamaModel { let path_str = path .to_str() .ok_or_else(|| LlamaLoraAdapterInitError::PathToStrError(path.to_path_buf()))?; + let cstr = CString::new(path_str)?; if !path.exists() { return Err(LlamaLoraAdapterInitError::FileNotFound(path.to_path_buf())); } - - let cstr = CString::new(path_str)?; initialize_lora_adapter(|| unsafe { llama_cpp_bindings_sys::llama_adapter_lora_init(self.model.as_ptr(), cstr.as_ptr()) }) @@ -635,19 +645,14 @@ impl LlamaModel { } } -const TOK_ENV_PIECE_PROBE_SIZE: usize = 32; - fn token_piece_bytes_for_tok_env( model: &LlamaModel, token: LlamaToken, special: bool, ) -> Result, TokenToStringError> { - match model.token_to_piece_bytes(token, TOK_ENV_PIECE_PROBE_SIZE, special, None) { + match model.token_to_piece_bytes_fitted(token, special, None) { Err(TokenToStringError::UnknownTokenType) => Ok(Vec::new()), - Err(TokenToStringError::InsufficientBufferSpace(required)) => { - model.token_to_piece_bytes(token, required.unsigned_abs() as usize, special, None) - } - probe_result => probe_result, + piece_result => piece_result, } } @@ -801,8 +806,8 @@ fn tokenize_into_buffer( c_int, ) -> Result, ) -> Result, StringToTokenError> { + let buffer_capacity = checked_token_buffer_capacity(estimated_capacity)?; let mut buffer: Vec = Vec::with_capacity(estimated_capacity); - let buffer_capacity = checked_token_buffer_capacity(buffer.capacity())?; let size = invoke( buffer @@ -812,12 +817,14 @@ fn tokenize_into_buffer( )?; let size = if size.is_negative() { - buffer.reserve_exact(checked_token_count(-size)?); + let required_size = size.wrapping_neg(); + + buffer.reserve_exact(checked_token_count(required_size)?); invoke( buffer .as_mut_ptr() .cast::(), - -size, + required_size, )? } else { size @@ -971,6 +978,26 @@ mod tokenize_into_buffer_tests { assert_eq!(call_count.get(), 2); } + #[test] + fn tokenize_into_buffer_refuses_a_capacity_beyond_a_c_int_before_allocating() { + assert_eq!( + tokenize_into_buffer(usize::MAX, |_tokens, _n_tokens_max| Ok(0)), + Err(crate::StringToTokenError::CIntConversionError( + i32::try_from(usize::MAX).unwrap_err() + )) + ); + } + + #[test] + fn tokenize_into_buffer_refuses_a_required_size_it_cannot_negate() { + assert_eq!( + tokenize_into_buffer(8, |_tokens, _n_tokens_max| Ok(i32::MIN)), + Err(crate::StringToTokenError::CIntConversionError( + usize::try_from(i32::MIN).unwrap_err() + )) + ); + } + #[test] fn tokenize_into_buffer_negative_final_size_returns_conversion_error() { let call_count = std::cell::Cell::new(0); diff --git a/llama-cpp-bindings/src/model/params.rs b/llama-cpp-bindings/src/model/params.rs index c651e3fe7..3775b7834 100644 --- a/llama-cpp-bindings/src/model/params.rs +++ b/llama-cpp-bindings/src/model/params.rs @@ -301,27 +301,17 @@ impl LlamaModelParams { #[must_use] pub fn devices(&self) -> Vec { - let mut backend_devices = Vec::new(); - for device_index in 0..unsafe { llama_cpp_bindings_sys::ggml_backend_dev_count() } { - let dev = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_get(device_index) }; - backend_devices.push(dev); - } - let mut devices = Vec::new(); - for &dev in self.devices.iter() { - if dev.is_null() { - break; - } - let matched_index = backend_devices - .iter() - .enumerate() - .find(|&(_i, &d)| d == dev) - .map(|(index, _)| index); - - if let Some(index) = matched_index { - devices.push(index); - } - } - devices + let device_count = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_count() }; + + self.devices + .iter() + .take_while(|device| !device.is_null()) + .filter_map(|device| { + (0..device_count).position(|device_index| { + *device == unsafe { llama_cpp_bindings_sys::ggml_backend_dev_get(device_index) } + }) + }) + .collect() } #[must_use] @@ -913,10 +903,7 @@ mod tests { llama_cpp_bindings_sys::GGML_LOG_LEVEL_NONE, ); - assert!( - matches!(result, Err(FitError::Aborted | FitError::Reported { .. })), - "expected Aborted or Reported, got {result:?}" - ); + assert_eq!(result.err(), Some(FitError::Aborted)); } #[test] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs index 2516cb7bc..569f2bbe7 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs @@ -149,7 +149,7 @@ pub struct MtmdBitmap { pub bitmap: NonNull, } -const RGB_BYTES_PER_PIXEL: usize = 3; +const RGB_BYTES_PER_PIXEL: u64 = 3; unsafe impl Send for MtmdBitmap {} unsafe impl Sync for MtmdBitmap {} @@ -167,10 +167,9 @@ impl MtmdBitmap { return Err(MtmdBitmapError::ImageDimensionsTooSmall(nx, ny)); } - let expected_len = usize::try_from(nx) - .ok() - .and_then(|width| width.checked_mul(usize::try_from(ny).ok()?)) - .and_then(|pixels| pixels.checked_mul(RGB_BYTES_PER_PIXEL)) + let expected_len = (u64::from(nx) * u64::from(ny)) + .checked_mul(RGB_BYTES_PER_PIXEL) + .and_then(|bytes| usize::try_from(bytes).ok()) .ok_or(MtmdBitmapError::ImageDimensionsOverflow { nx, ny })?; if data.len() != expected_len { diff --git a/llama-cpp-bindings/src/mtmd/mtmd_default_marker.rs b/llama-cpp-bindings/src/mtmd/mtmd_default_marker.rs index facff884a..33e7186a1 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_default_marker.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_default_marker.rs @@ -22,15 +22,15 @@ pub fn mtmd_default_marker() -> Result<&'static str, MtmdDefaultMarkerError> { #[cfg(test)] mod tests { use std::os::raw::c_char; + use std::str; use super::marker_bytes_to_str; use super::mtmd_default_marker; use crate::mtmd::mtmd_default_marker_error::MtmdDefaultMarkerError; #[test] - fn returns_non_empty_marker() { - let marker = mtmd_default_marker().expect("llama.cpp marker must be valid UTF-8"); - assert_ne!(marker, ""); + fn the_default_marker_is_the_media_marker() { + assert_eq!(mtmd_default_marker(), Ok("<__media__>")); } #[test] @@ -38,6 +38,11 @@ mod tests { let invalid: [u8; 3] = [0xFF, 0xFE, 0x00]; let result = unsafe { marker_bytes_to_str(invalid.as_ptr().cast::()) }; - assert!(matches!(result, Err(MtmdDefaultMarkerError::NotUtf8(_)))); + assert_eq!( + result.err(), + str::from_utf8(&invalid[..2]) + .err() + .map(MtmdDefaultMarkerError::NotUtf8) + ); } } diff --git a/llama-cpp-bindings/src/sampling.rs b/llama-cpp-bindings/src/sampling.rs index 7d1a4b27c..288b68c6b 100644 --- a/llama-cpp-bindings/src/sampling.rs +++ b/llama-cpp-bindings/src/sampling.rs @@ -2016,6 +2016,25 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { ); } + #[test] + fn an_invalid_trigger_pattern_without_a_message_is_a_contract_error() { + assert_eq!( + sampler_init_grammar_lazy_patterns_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_INVALID_TRIGGER_PATTERN, + ptr::null_mut(), + ptr::null_mut(), + ) + .err(), + Some( + crate::FfiContractError { + operation: "llama_rs_sampler_init_grammar_lazy_patterns", + detail: "reported a thrown C++ exception without an error message", + } + .into() + ) + ); + } + #[test] fn lazy_grammar_null_output_argument_status_is_a_contract_error() { let status = llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_NULL_OUT_SAMPLER_ARG; diff --git a/llama-cpp-bindings/src/send_logs_to_log.rs b/llama-cpp-bindings/src/send_logs_to_log.rs index 93f1a005b..f4e94b736 100644 --- a/llama-cpp-bindings/src/send_logs_to_log.rs +++ b/llama-cpp-bindings/src/send_logs_to_log.rs @@ -218,9 +218,8 @@ mod tests { fn ensure_test_logger_installed() { INSTALL.call_once(|| { - if log::set_logger(&TEST_LOGGER).is_ok() { - log::set_max_level(log::LevelFilter::Trace); - } + log::set_logger(&TEST_LOGGER).unwrap(); + log::set_max_level(log::LevelFilter::Trace); }); } diff --git a/llama-cpp-bindings/src/streaming_json_probe.rs b/llama-cpp-bindings/src/streaming_json_probe.rs index d48241b87..9be943071 100644 --- a/llama-cpp-bindings/src/streaming_json_probe.rs +++ b/llama-cpp-bindings/src/streaming_json_probe.rs @@ -438,6 +438,11 @@ mod tests { assert_eq!(probe(r#"{"foo":"bar"}"#), JsonProbeOutcome::Failed,); } + #[test] + fn a_key_with_an_invalid_escape_is_failed() { + assert_eq!(probe(r#"{"\q":"bar"}"#), JsonProbeOutcome::Failed); + } + #[test] fn complete_object_with_non_string_name_is_failed() { assert_eq!( diff --git a/llama-cpp-bindings/src/token/data_array.rs b/llama-cpp-bindings/src/token/data_array.rs index 1fb73370e..df5362e11 100644 --- a/llama-cpp-bindings/src/token/data_array.rs +++ b/llama-cpp-bindings/src/token/data_array.rs @@ -222,6 +222,7 @@ impl LlamaTokenDataArray { mod tests { use super::LlamaTokenDataArray; use crate::error::SamplerApplyError; + use crate::error::TokenSamplingError; use crate::token::LlamaToken; use crate::token::data::LlamaTokenData; @@ -252,6 +253,93 @@ mod tests { ); } + fn excluded_candidates() -> LlamaTokenDataArray { + LlamaTokenDataArray::new( + vec![ + LlamaTokenData::new(LlamaToken::new(0), f32::NEG_INFINITY, 0.0), + LlamaTokenData::new(LlamaToken::new(1), f32::NEG_INFINITY, 0.0), + ], + false, + ) + } + + unsafe extern "C" fn claim_more_candidates_than_exist( + _sampler: *mut llama_cpp_bindings_sys::llama_sampler, + candidates: *mut llama_cpp_bindings_sys::llama_token_data_array, + ) { + unsafe { (*candidates).size = usize::MAX }; + } + + static mut OVERREPORTING_SAMPLER: llama_cpp_bindings_sys::llama_sampler_i = + llama_cpp_bindings_sys::llama_sampler_i { + name: None, + accept: None, + apply: Some(claim_more_candidates_than_exist), + reset: None, + clone: None, + free: None, + backend_init: None, + backend_accept: None, + backend_apply: None, + backend_set_input: None, + backend_reset: None, + copy_state: None, + }; + + #[test] + fn a_sampler_claiming_more_candidates_than_exist_is_a_contract_violation() { + use crate::sampling::LlamaSampler; + + let overreporting_sampler = + LlamaSampler::initialize("overreporting", |out_sampler, out_error| unsafe { + llama_cpp_bindings_sys::llama_rs_sampler_init( + &raw mut OVERREPORTING_SAMPLER, + std::ptr::null_mut(), + out_sampler, + out_error, + ) + }) + .unwrap(); + + assert_eq!( + excluded_candidates().apply_sampler(&overreporting_sampler), + Err(SamplerApplyError::FfiContract(crate::FfiContractError { + operation: "modify_as_c_llama_token_data_array", + detail: "the llama.cpp sampler grew the token data array beyond its capacity", + })) + ); + } + + #[test] + fn building_candidates_with_a_sampler_that_excludes_all_of_them_fails() { + use crate::sampling::LlamaSampler; + + assert_eq!( + excluded_candidates() + .with_sampler(&mut LlamaSampler::greedy().unwrap()) + .err(), + Some(SamplerApplyError::EveryCandidateExcluded) + ); + } + + #[test] + fn sampling_candidates_that_are_all_excluded_selects_no_token() { + assert_eq!( + [ + excluded_candidates().sample_token(7), + excluded_candidates().sample_token_greedy(), + ], + [ + Err(TokenSamplingError::SamplerApply( + SamplerApplyError::EveryCandidateExcluded + )), + Err(TokenSamplingError::SamplerApply( + SamplerApplyError::EveryCandidateExcluded + )), + ] + ); + } + #[test] fn apply_greedy_sampler_selects_highest_logit() { use crate::sampling::LlamaSampler; From 5a221cb348c6877652f2f355c9da96903c8c2233 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Fri, 30 Oct 2026 14:21:19 +0100 Subject: [PATCH 10/82] Surface C++ exceptions from mtmd bitmap, chunk list and chunk copy allocations as typed errors --- llama-cpp-bindings-sys/wrapper_mtmd.cpp | 111 +++++++ llama-cpp-bindings-sys/wrapper_mtmd.h | 57 ++++ llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs | 279 +++++++++++++++++- .../src/mtmd/mtmd_input_chunk.rs | 144 ++++++++- .../src/mtmd/mtmd_input_chunk_error.rs | 14 +- .../src/mtmd/mtmd_input_chunks.rs | 136 ++++++++- .../src/mtmd/mtmd_input_chunks_error.rs | 12 +- 7 files changed, 726 insertions(+), 27 deletions(-) diff --git a/llama-cpp-bindings-sys/wrapper_mtmd.cpp b/llama-cpp-bindings-sys/wrapper_mtmd.cpp index 33ff45148..107e7713c 100644 --- a/llama-cpp-bindings-sys/wrapper_mtmd.cpp +++ b/llama-cpp-bindings-sys/wrapper_mtmd.cpp @@ -367,3 +367,114 @@ extern "C" auto llama_rs_mtmd_get_memory_usage( LLAMA_RS_MTMD_GET_MEMORY_USAGE_ERROR_STRING_ALLOCATION_FAILED); } } + +extern "C" auto llama_rs_mtmd_bitmap_init( + uint32_t width, + uint32_t height, + const unsigned char * data, + struct mtmd_bitmap ** out_bitmap, + char ** out_error) -> llama_rs_mtmd_bitmap_init_status { + if (out_error != nullptr) { + *out_error = nullptr; + } + if (out_bitmap == nullptr) { + return LLAMA_RS_MTMD_BITMAP_INIT_NULL_OUT_BITMAP_ARG; + } + *out_bitmap = nullptr; + if (data == nullptr) { + return LLAMA_RS_MTMD_BITMAP_INIT_NULL_DATA_ARG; + } + + try { + *out_bitmap = mtmd_bitmap_init(width, height, data); + + return LLAMA_RS_MTMD_BITMAP_INIT_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_MTMD_BITMAP_INIT_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_MTMD_BITMAP_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_MTMD_BITMAP_INIT_ERROR_STRING_ALLOCATION_FAILED); + } +} + +extern "C" auto llama_rs_mtmd_bitmap_init_from_audio( + size_t n_samples, + const float * data, + struct mtmd_bitmap ** out_bitmap, + char ** out_error) -> llama_rs_mtmd_bitmap_init_from_audio_status { + if (out_error != nullptr) { + *out_error = nullptr; + } + if (out_bitmap == nullptr) { + return LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_OUT_BITMAP_ARG; + } + *out_bitmap = nullptr; + if (data == nullptr) { + return LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_DATA_ARG; + } + + try { + *out_bitmap = mtmd_bitmap_init_from_audio(n_samples, data); + + return LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_ERROR_STRING_ALLOCATION_FAILED); + } +} + +extern "C" auto llama_rs_mtmd_input_chunks_init( + struct mtmd_input_chunks ** out_chunks, + char ** out_error) -> llama_rs_mtmd_input_chunks_init_status { + if (out_error != nullptr) { + *out_error = nullptr; + } + if (out_chunks == nullptr) { + return LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_NULL_OUT_CHUNKS_ARG; + } + *out_chunks = nullptr; + + try { + *out_chunks = mtmd_input_chunks_init(); + + return LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_ERROR_STRING_ALLOCATION_FAILED); + } +} + +extern "C" auto llama_rs_mtmd_input_chunk_copy( + const struct mtmd_input_chunk * chunk, + struct mtmd_input_chunk ** out_chunk, + char ** out_error) -> llama_rs_mtmd_input_chunk_copy_status { + if (out_error != nullptr) { + *out_error = nullptr; + } + if (out_chunk == nullptr) { + return LLAMA_RS_MTMD_INPUT_CHUNK_COPY_NULL_OUT_CHUNK_ARG; + } + *out_chunk = nullptr; + if (chunk == nullptr) { + return LLAMA_RS_MTMD_INPUT_CHUNK_COPY_NULL_CHUNK_ARG; + } + + try { + *out_chunk = mtmd_input_chunk_copy(chunk); + + return LLAMA_RS_MTMD_INPUT_CHUNK_COPY_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_MTMD_INPUT_CHUNK_COPY_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_MTMD_INPUT_CHUNK_COPY_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_MTMD_INPUT_CHUNK_COPY_ERROR_STRING_ALLOCATION_FAILED); + } +} diff --git a/llama-cpp-bindings-sys/wrapper_mtmd.h b/llama-cpp-bindings-sys/wrapper_mtmd.h index 86393586d..09aa0ea18 100644 --- a/llama-cpp-bindings-sys/wrapper_mtmd.h +++ b/llama-cpp-bindings-sys/wrapper_mtmd.h @@ -178,6 +178,63 @@ llama_rs_mtmd_get_memory_usage_status llama_rs_mtmd_get_memory_usage( bool * out_use_non_causal, char ** out_error); +typedef enum llama_rs_mtmd_bitmap_init_status { + LLAMA_RS_MTMD_BITMAP_INIT_OK = 0, + LLAMA_RS_MTMD_BITMAP_INIT_NULL_DATA_ARG, + LLAMA_RS_MTMD_BITMAP_INIT_NULL_OUT_BITMAP_ARG, + LLAMA_RS_MTMD_BITMAP_INIT_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_MTMD_BITMAP_INIT_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_MTMD_BITMAP_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_mtmd_bitmap_init_status; + +llama_rs_mtmd_bitmap_init_status llama_rs_mtmd_bitmap_init( + uint32_t width, + uint32_t height, + const unsigned char * data, + struct mtmd_bitmap ** out_bitmap, + char ** out_error); + +typedef enum llama_rs_mtmd_bitmap_init_from_audio_status { + LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_OK = 0, + LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_DATA_ARG, + LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_OUT_BITMAP_ARG, + LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_mtmd_bitmap_init_from_audio_status; + +llama_rs_mtmd_bitmap_init_from_audio_status llama_rs_mtmd_bitmap_init_from_audio( + size_t n_samples, + const float * data, + struct mtmd_bitmap ** out_bitmap, + char ** out_error); + +typedef enum llama_rs_mtmd_input_chunks_init_status { + LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_OK = 0, + LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_NULL_OUT_CHUNKS_ARG, + LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_mtmd_input_chunks_init_status; + +llama_rs_mtmd_input_chunks_init_status llama_rs_mtmd_input_chunks_init( + struct mtmd_input_chunks ** out_chunks, + char ** out_error); + +typedef enum llama_rs_mtmd_input_chunk_copy_status { + LLAMA_RS_MTMD_INPUT_CHUNK_COPY_OK = 0, + LLAMA_RS_MTMD_INPUT_CHUNK_COPY_NULL_CHUNK_ARG, + LLAMA_RS_MTMD_INPUT_CHUNK_COPY_NULL_OUT_CHUNK_ARG, + LLAMA_RS_MTMD_INPUT_CHUNK_COPY_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_MTMD_INPUT_CHUNK_COPY_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_MTMD_INPUT_CHUNK_COPY_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_mtmd_input_chunk_copy_status; + +llama_rs_mtmd_input_chunk_copy_status llama_rs_mtmd_input_chunk_copy( + const struct mtmd_input_chunk * chunk, + struct mtmd_input_chunk ** out_chunk, + char ** out_error); + #ifdef __cplusplus } #endif diff --git a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs index 569f2bbe7..9c86c9e54 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs @@ -137,6 +137,114 @@ unsafe fn from_buf_status_to_result( } } +fn bitmap_init_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_mtmd_bitmap_init_status, + out_bitmap: *mut llama_cpp_bindings_sys::mtmd_bitmap, + out_error: *mut c_char, +) -> Result { + const OPERATION: &str = "llama_rs_mtmd_bitmap_init"; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_OK => NonNull::new(out_bitmap) + .map(|bitmap| MtmdBitmap { bitmap }) + .ok_or_else(|| { + crate::FfiContractError { + operation: OPERATION, + detail: "success status contained a null bitmap", + } + .into() + }), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_NULL_DATA_ARG => { + Err(crate::FfiContractError { + operation: OPERATION, + detail: "was given a null data argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_NULL_OUT_BITMAP_ARG => { + Err(crate::FfiContractError { + operation: OPERATION, + detail: "was given a null out_bitmap argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_ERROR_STRING_ALLOCATION_FAILED => { + Err(MtmdBitmapError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_LLAMA_CPP_OUT_OF_MEMORY => { + Err(MtmdBitmapError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + OPERATION, + "reported a thrown C++ exception without an error message", + ) + }?; + + Err(MtmdBitmapError::Reported { message }) + } + other => Err(crate::FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), + } +} + +fn bitmap_init_from_audio_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_mtmd_bitmap_init_from_audio_status, + out_bitmap: *mut llama_cpp_bindings_sys::mtmd_bitmap, + out_error: *mut c_char, +) -> Result { + const OPERATION: &str = "llama_rs_mtmd_bitmap_init_from_audio"; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_OK => NonNull::new(out_bitmap) + .map(|bitmap| MtmdBitmap { bitmap }) + .ok_or_else(|| { + crate::FfiContractError { + operation: OPERATION, + detail: "success status contained a null bitmap", + } + .into() + }), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_DATA_ARG => Err(crate::FfiContractError { + operation: OPERATION, + detail: "was given a null data argument", + } + .into()), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_OUT_BITMAP_ARG => Err(crate::FfiContractError { + operation: OPERATION, + detail: "was given a null out_bitmap argument", + } + .into()), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_ERROR_STRING_ALLOCATION_FAILED => { + Err(MtmdBitmapError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_LLAMA_CPP_OUT_OF_MEMORY => { + Err(MtmdBitmapError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + OPERATION, + "reported a thrown C++ exception without an error message", + ) + }?; + + Err(MtmdBitmapError::Reported { message }) + } + other => Err(crate::FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), + } +} + /// Owns the bitmap and frees it on drop, so it cannot be cloned. /// /// ```compile_fail,E0277 @@ -160,7 +268,7 @@ impl MtmdBitmap { /// * `ImageDimensionsTooSmall` - `nx` or `ny` is below the 2x2 minimum /// * `ImageDimensionsOverflow` - `nx * ny * 3` does not fit into a buffer length /// * `InvalidDataSize` - Data length doesn't match `nx * ny * 3` - /// * `BitmapDecodeFailed` - Underlying C function returned null + /// * the [`MtmdBitmapError`] variant matching the wrapper's status code /// pub fn from_image_data(nx: u32, ny: u32, data: &[u8]) -> Result { if nx < 2 || ny < 2 { @@ -176,25 +284,37 @@ impl MtmdBitmap { return Err(MtmdBitmapError::InvalidDataSize); } - let bitmap = unsafe { llama_cpp_bindings_sys::mtmd_bitmap_init(nx, ny, data.as_ptr()) }; - - let bitmap = NonNull::new(bitmap).ok_or(MtmdBitmapError::BitmapDecodeFailed)?; + let mut out_bitmap: *mut llama_cpp_bindings_sys::mtmd_bitmap = std::ptr::null_mut(); + let mut out_error: *mut c_char = std::ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_mtmd_bitmap_init( + nx, + ny, + data.as_ptr(), + &raw mut out_bitmap, + &raw mut out_error, + ) + }; - Ok(Self { bitmap }) + bitmap_init_status_to_result(status, out_bitmap, out_error) } /// # Errors /// - /// * `NullResult` - Underlying C function returned null - /// + /// Returns the [`MtmdBitmapError`] variant matching the wrapper's status code. pub fn from_audio_data(data: &[f32]) -> Result { - let bitmap = unsafe { - llama_cpp_bindings_sys::mtmd_bitmap_init_from_audio(data.len(), data.as_ptr()) + let mut out_bitmap: *mut llama_cpp_bindings_sys::mtmd_bitmap = std::ptr::null_mut(); + let mut out_error: *mut c_char = std::ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_mtmd_bitmap_init_from_audio( + data.len(), + data.as_ptr(), + &raw mut out_bitmap, + &raw mut out_error, + ) }; - let bitmap = NonNull::new(bitmap).ok_or(MtmdBitmapError::BitmapDecodeFailed)?; - - Ok(Self { bitmap }) + bitmap_init_from_audio_status_to_result(status, out_bitmap, out_error) } /// # Errors @@ -814,3 +934,138 @@ mod from_buf_status_to_result_tests { ); } } + +#[cfg(test)] +mod bitmap_init_status_to_result_tests { + use std::ptr; + + use super::bitmap_init_from_audio_status_to_result; + use super::bitmap_init_status_to_result; + use crate::mtmd::mtmd_bitmap_error::MtmdBitmapError; + + fn contract_violation(operation: &'static str, detail: &'static str) -> MtmdBitmapError { + crate::FfiContractError { operation, detail }.into() + } + + #[test] + fn every_image_failure_status_maps_to_its_error() { + let statuses = [ + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_OK, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_NULL_DATA_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_NULL_OUT_BITMAP_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_ERROR_STRING_ALLOCATION_FAILED, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_LLAMA_CPP_OUT_OF_MEMORY, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + 255, + ]; + let operation = "llama_rs_mtmd_bitmap_init"; + + assert_eq!( + statuses.map(|status| { + bitmap_init_status_to_result(status, ptr::null_mut(), ptr::null_mut()).err() + }), + [ + Some(contract_violation( + operation, + "success status contained a null bitmap" + )), + Some(contract_violation( + operation, + "was given a null data argument" + )), + Some(contract_violation( + operation, + "was given a null out_bitmap argument" + )), + Some(MtmdBitmapError::NotEnoughMemory), + Some(MtmdBitmapError::LlamaCppOutOfMemory), + Some(contract_violation( + operation, + "reported a thrown C++ exception without an error message" + )), + Some(MtmdBitmapError::FfiStatus(crate::FfiStatusError { + operation, + code: 255, + })), + ] + ); + } + + #[test] + fn every_audio_failure_status_maps_to_its_error() { + let statuses = [ + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_OK, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_DATA_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_OUT_BITMAP_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_ERROR_STRING_ALLOCATION_FAILED, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_LLAMA_CPP_OUT_OF_MEMORY, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_LLAMA_CPP_THREW_CXX_EXCEPTION, + 255, + ]; + let operation = "llama_rs_mtmd_bitmap_init_from_audio"; + + assert_eq!( + statuses.map(|status| { + bitmap_init_from_audio_status_to_result(status, ptr::null_mut(), ptr::null_mut()) + .err() + }), + [ + Some(contract_violation( + operation, + "success status contained a null bitmap" + )), + Some(contract_violation( + operation, + "was given a null data argument" + )), + Some(contract_violation( + operation, + "was given a null out_bitmap argument" + )), + Some(MtmdBitmapError::NotEnoughMemory), + Some(MtmdBitmapError::LlamaCppOutOfMemory), + Some(contract_violation( + operation, + "reported a thrown C++ exception without an error message" + )), + Some(MtmdBitmapError::FfiStatus(crate::FfiStatusError { + operation, + code: 255, + })), + ] + ); + } + + #[test] + fn thrown_exceptions_report_their_messages() { + let image_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"image threw".as_ptr()) }; + let audio_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"audio threw".as_ptr()) }; + + assert_eq!( + [ + bitmap_init_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + image_error, + ) + .err(), + bitmap_init_from_audio_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + audio_error, + ) + .err(), + ], + [ + Some(MtmdBitmapError::Reported { + message: "image threw".to_owned(), + }), + Some(MtmdBitmapError::Reported { + message: "audio threw".to_owned(), + }), + ] + ); + } +} diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs index 4c38746bc..52387656a 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs @@ -1,7 +1,11 @@ use std::ffi::CStr; +use std::ffi::c_char; +use std::ptr; use std::ptr::NonNull; use std::slice; +use llama_cpp_ffi_status::read_and_free_cpp_string; + use crate::token::LlamaToken; use super::mtmd_context::MtmdContext; @@ -50,6 +54,62 @@ fn non_causal_chunk_micro_batch_error( None } +fn input_chunk_copy_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_mtmd_input_chunk_copy_status, + out_chunk: *mut llama_cpp_bindings_sys::mtmd_input_chunk, + out_error: *mut c_char, +) -> Result { + const OPERATION: &str = "llama_rs_mtmd_input_chunk_copy"; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_OK => NonNull::new(out_chunk) + .map(|chunk| MtmdInputChunk { chunk, owned: true }) + .ok_or_else(|| { + crate::FfiContractError { + operation: OPERATION, + detail: "success status contained a null chunk", + } + .into() + }), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_NULL_CHUNK_ARG => { + Err(crate::FfiContractError { + operation: OPERATION, + detail: "was given a null chunk argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_NULL_OUT_CHUNK_ARG => { + Err(crate::FfiContractError { + operation: OPERATION, + detail: "was given a null out_chunk argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_ERROR_STRING_ALLOCATION_FAILED => { + Err(MtmdInputChunkError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_LLAMA_CPP_OUT_OF_MEMORY => { + Err(MtmdInputChunkError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + OPERATION, + "reported a thrown C++ exception without an error message", + ) + }?; + + Err(MtmdInputChunkError::Reported { message }) + } + other => Err(crate::FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), + } +} + #[derive(Debug)] pub struct MtmdInputChunk { pub chunk: NonNull, @@ -107,12 +167,19 @@ impl MtmdInputChunk { /// # Errors /// - /// Returns `MtmdInputChunkError::ChunkOperationFailed` if copying fails. + /// Returns the [`MtmdInputChunkError`] variant matching the wrapper's status code. pub fn copy(&self) -> Result { - let chunk = unsafe { llama_cpp_bindings_sys::mtmd_input_chunk_copy(self.chunk.as_ptr()) }; - let chunk = NonNull::new(chunk).ok_or(MtmdInputChunkError::ChunkOperationFailed)?; + let mut out_chunk: *mut llama_cpp_bindings_sys::mtmd_input_chunk = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_mtmd_input_chunk_copy( + self.chunk.as_ptr(), + &raw mut out_chunk, + &raw mut out_error, + ) + }; - Ok(Self { chunk, owned: true }) + input_chunk_copy_status_to_result(status, out_chunk, out_error) } /// Checks that this chunk can be evaluated in decodes of `micro_batch_tokens` and returns the @@ -200,3 +267,72 @@ mod unit_tests { assert!(non_causal_chunk_micro_batch_error(true, 4, 4).is_none()); } } + +#[cfg(test)] +mod input_chunk_copy_status_to_result_tests { + use std::ptr; + + use super::input_chunk_copy_status_to_result; + use crate::mtmd::mtmd_input_chunk_error::MtmdInputChunkError; + + const OPERATION: &str = "llama_rs_mtmd_input_chunk_copy"; + + fn contract_violation(detail: &'static str) -> MtmdInputChunkError { + crate::FfiContractError { + operation: OPERATION, + detail, + } + .into() + } + + #[test] + fn every_failure_status_maps_to_its_error() { + let statuses = [ + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_OK, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_NULL_CHUNK_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_NULL_OUT_CHUNK_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_ERROR_STRING_ALLOCATION_FAILED, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_LLAMA_CPP_OUT_OF_MEMORY, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_LLAMA_CPP_THREW_CXX_EXCEPTION, + 255, + ]; + + assert_eq!( + statuses.map(|status| { + input_chunk_copy_status_to_result(status, ptr::null_mut(), ptr::null_mut()).err() + }), + [ + Some(contract_violation("success status contained a null chunk")), + Some(contract_violation("was given a null chunk argument")), + Some(contract_violation("was given a null out_chunk argument")), + Some(MtmdInputChunkError::NotEnoughMemory), + Some(MtmdInputChunkError::LlamaCppOutOfMemory), + Some(contract_violation( + "reported a thrown C++ exception without an error message" + )), + Some(MtmdInputChunkError::FfiStatus(crate::FfiStatusError { + operation: OPERATION, + code: 255, + })), + ] + ); + } + + #[test] + fn a_thrown_exception_reports_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"allocation threw".as_ptr()) }; + + assert_eq!( + input_chunk_copy_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + out_error, + ) + .err(), + Some(MtmdInputChunkError::Reported { + message: "allocation threw".to_owned(), + }) + ); + } +} diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_error.rs index ab94f14b3..5a72a80a0 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_error.rs @@ -1,5 +1,13 @@ -#[derive(thiserror::Error, Debug)] +#[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdInputChunkError { - #[error("input chunk operation failed")] - ChunkOperationFailed, + #[error(transparent)] + FfiStatus(#[from] crate::FfiStatusError), + #[error(transparent)] + FfiContract(#[from] crate::FfiContractError), + #[error("not enough memory")] + NotEnoughMemory, + #[error("the llama.cpp library ran out of memory")] + LlamaCppOutOfMemory, + #[error("{message}")] + Reported { message: String }, } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs index ace152a16..74ef98245 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs @@ -1,5 +1,9 @@ +use std::ffi::c_char; +use std::ptr; use std::ptr::NonNull; +use llama_cpp_ffi_status::read_and_free_cpp_string; + use crate::context::LlamaContext; use crate::context::decoded_output_rows::DecodedOutputRows; @@ -18,6 +22,55 @@ const fn check_eval_result(result: i32) -> Result<(), MtmdEvalError> { } } +fn input_chunks_init_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_mtmd_input_chunks_init_status, + out_chunks: *mut llama_cpp_bindings_sys::mtmd_input_chunks, + out_error: *mut c_char, +) -> Result { + const OPERATION: &str = "llama_rs_mtmd_input_chunks_init"; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_OK => NonNull::new(out_chunks) + .map(|chunks| MtmdInputChunks { chunks }) + .ok_or_else(|| { + crate::FfiContractError { + operation: OPERATION, + detail: "success status contained null chunks", + } + .into() + }), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_NULL_OUT_CHUNKS_ARG => { + Err(crate::FfiContractError { + operation: OPERATION, + detail: "was given a null out_chunks argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_ERROR_STRING_ALLOCATION_FAILED => { + Err(MtmdInputChunksError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_LLAMA_CPP_OUT_OF_MEMORY => { + Err(MtmdInputChunksError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + OPERATION, + "reported a thrown C++ exception without an error message", + ) + }?; + + Err(MtmdInputChunksError::Reported { message }) + } + other => Err(crate::FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), + } +} + #[derive(Debug)] pub struct MtmdInputChunks { pub chunks: NonNull, @@ -28,14 +81,18 @@ unsafe impl Send for MtmdInputChunks {} impl MtmdInputChunks { /// # Errors /// - /// Returns `MtmdInputChunksError::ChunksCreationFailed` if the underlying llama.cpp function - /// returns null. - /// + /// Returns the [`MtmdInputChunksError`] variant matching the wrapper's status code. pub fn new() -> Result { - let chunks = unsafe { llama_cpp_bindings_sys::mtmd_input_chunks_init() }; - let chunks = NonNull::new(chunks).ok_or(MtmdInputChunksError::ChunksCreationFailed)?; + let mut out_chunks: *mut llama_cpp_bindings_sys::mtmd_input_chunks = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_mtmd_input_chunks_init( + &raw mut out_chunks, + &raw mut out_error, + ) + }; - Ok(Self { chunks }) + input_chunks_init_status_to_result(status, out_chunks, out_error) } #[must_use] @@ -193,3 +250,70 @@ mod tests { ); } } + +#[cfg(test)] +mod input_chunks_init_status_to_result_tests { + use std::ptr; + + use super::input_chunks_init_status_to_result; + use crate::mtmd::mtmd_input_chunks_error::MtmdInputChunksError; + + const OPERATION: &str = "llama_rs_mtmd_input_chunks_init"; + + fn contract_violation(detail: &'static str) -> MtmdInputChunksError { + crate::FfiContractError { + operation: OPERATION, + detail, + } + .into() + } + + #[test] + fn every_failure_status_maps_to_its_error() { + let statuses = [ + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_OK, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_NULL_OUT_CHUNKS_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_ERROR_STRING_ALLOCATION_FAILED, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_LLAMA_CPP_OUT_OF_MEMORY, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + 255, + ]; + + assert_eq!( + statuses.map(|status| { + input_chunks_init_status_to_result(status, ptr::null_mut(), ptr::null_mut()).err() + }), + [ + Some(contract_violation("success status contained null chunks")), + Some(contract_violation("was given a null out_chunks argument")), + Some(MtmdInputChunksError::NotEnoughMemory), + Some(MtmdInputChunksError::LlamaCppOutOfMemory), + Some(contract_violation( + "reported a thrown C++ exception without an error message" + )), + Some(MtmdInputChunksError::FfiStatus(crate::FfiStatusError { + operation: OPERATION, + code: 255, + })), + ] + ); + } + + #[test] + fn a_thrown_exception_reports_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"allocation threw".as_ptr()) }; + + assert_eq!( + input_chunks_init_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + out_error, + ) + .err(), + Some(MtmdInputChunksError::Reported { + message: "allocation threw".to_owned(), + }) + ); + } +} diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks_error.rs index 0a8947e0d..195a5c62d 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks_error.rs @@ -1,5 +1,13 @@ #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdInputChunksError { - #[error("input chunks collection could not be created")] - ChunksCreationFailed, + #[error(transparent)] + FfiStatus(#[from] crate::FfiStatusError), + #[error(transparent)] + FfiContract(#[from] crate::FfiContractError), + #[error("not enough memory")] + NotEnoughMemory, + #[error("the llama.cpp library ran out of memory")] + LlamaCppOutOfMemory, + #[error("{message}")] + Reported { message: String }, } From 9de7103e18dbe6ebc8dc3c8d9881250c2af82e70 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Fri, 30 Oct 2026 18:58:28 +0100 Subject: [PATCH 11/82] Inject faults into the C++ wrappers in tests to cover every failure path, and make test helpers fail fast --- Cargo.lock | 9 + Cargo.toml | 2 + llama-cpp-bindings-build/Cargo.toml | 1 + .../src/bindgen_config.rs | 6 + llama-cpp-bindings-build/src/cpp_wrapper.rs | 5 + llama-cpp-bindings-sys/Cargo.toml | 1 + llama-cpp-bindings-sys/wrapper.h | 1 + llama-cpp-bindings-sys/wrapper_batch.cpp | 3 + .../wrapper_chat_session.cpp | 26 +- llama-cpp-bindings-sys/wrapper_chat_session.h | 1 + llama-cpp-bindings-sys/wrapper_common.cpp | 23 +- llama-cpp-bindings-sys/wrapper_common.h | 8 + llama-cpp-bindings-sys/wrapper_context.cpp | 22 ++ llama-cpp-bindings-sys/wrapper_context.h | 10 + .../wrapper_fault_injection.cpp | 82 ++++++ .../wrapper_fault_injection.h | 68 +++++ llama-cpp-bindings-sys/wrapper_mtmd.cpp | 25 ++ llama-cpp-bindings-sys/wrapper_mtmd.h | 6 + .../wrapper_parsed_chat.cpp | 17 +- .../wrapper_sampler_init.cpp | 11 +- llama-cpp-bindings-tests/Cargo.toml | 1 + llama-cpp-bindings-tests/src/chat_answer.rs | 71 +++-- .../src/chat_answer_generation.rs | 31 ++- .../src/chunk_token_breakdown.rs | 23 +- .../src/classify_sample_loop.rs | 42 +-- .../src/classify_sample_loop_outcome.rs | 33 ++- .../src/closed_reasoning_chat_session.rs | 21 +- .../src/ingested_generation.rs | 29 +- llama-cpp-bindings-tests/src/lib.rs | 6 +- .../src/media_question_chat_session.rs | 9 +- .../src/penalized_nucleus_sampler.rs | 26 +- .../src/recognized_answer.rs | 23 +- .../src/weather_chat_session.rs | 34 +++ .../tests/chat_answer_parsing.rs | 42 ++- .../tests/chat_session_llama_cpp_failures.rs | 254 ++++++++++++++++++ .../tests/completion_usage_accounting.rs | 4 +- .../tests/context_creation_limits.rs | 26 ++ .../tests/decode_batch_rejection.rs | 33 +++ .../tests/duck_typed_tool_calls.rs | 2 +- .../tests/encode_batch_rejection.rs | 34 +++ .../tests/generation_end_classification.rs | 6 +- .../tests/grammar_sampler.rs | 24 +- .../tests/llguidance_sampler.rs | 60 +++++ .../tests/logits_retrieval.rs | 30 +++ .../tests/lora_adapters.rs | 36 +++ llama-cpp-bindings-tests/tests/main.rs | 1 + .../tests/multimodal_audio.rs | 2 +- .../tests/multimodal_chunk_evaluation.rs | 136 +++++++++- .../tests/multimodal_chunk_tokenization.rs | 39 +++ .../tests/multimodal_fusion.rs | 4 +- .../multimodal_prompt_parts_tokenization.rs | 58 ++++ .../multimodal_reasoning_classification.rs | 8 +- .../tests/multimodal_usage_accounting.rs | 2 +- .../tests/multimodal_vision_inference.rs | 4 +- .../tests/reasoning_classification.rs | 92 ++++--- .../tests/sampled_token_ingestion.rs | 106 ++++++++ .../tests/tool_call_classification.rs | 4 +- .../tests/tool_call_grammar.rs | 38 +-- .../tests/tool_call_parsing.rs | 12 +- llama-cpp-bindings/Cargo.toml | 1 + .../src/chat_message_parse_outcome.rs | 38 +++ llama-cpp-bindings/src/context.rs | 4 +- .../src/error/eval_multimodal_chunks_error.rs | 2 +- .../src/error/llama_cpp_error.rs | 2 +- .../src/lazy_grammar_triggers.rs | 16 ++ llama-cpp-bindings/src/llguidance_sampler.rs | 63 ++++- llama-cpp-bindings/src/model.rs | 34 ++- .../src/mtmd/mtmd_input_chunk.rs | 5 +- .../src/mtmd/mtmd_input_chunks.rs | 5 +- .../src/mtmd/mtmd_memory_usage.rs | 18 ++ llama-cpp-bindings/src/sampling.rs | 24 +- llama-cpp-bindings/src/token/data_array.rs | 40 +++ .../src/tool_call_format/json_object.rs | 6 +- llama-cpp-fault-injection/Cargo.toml | 14 + llama-cpp-fault-injection/src/armed_fault.rs | 36 +++ .../src/injected_fault_message.rs | 14 + llama-cpp-fault-injection/src/lib.rs | 5 + .../src/harness_run_error.rs | 2 +- llama-cpp-test-harness/src/run.rs | 8 +- .../src/compile_commands_file.rs | 17 +- .../src/fault_injection_define.rs | 1 + llama-cpp-wrapper-sources/src/lib.rs | 1 + .../src/wrapper_headers.rs | 1 + .../src/wrapper_sources.rs | 1 + 84 files changed, 1810 insertions(+), 281 deletions(-) create mode 100644 llama-cpp-bindings-sys/wrapper_fault_injection.cpp create mode 100644 llama-cpp-bindings-sys/wrapper_fault_injection.h create mode 100644 llama-cpp-bindings-tests/src/weather_chat_session.rs create mode 100644 llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs create mode 100644 llama-cpp-fault-injection/Cargo.toml create mode 100644 llama-cpp-fault-injection/src/armed_fault.rs create mode 100644 llama-cpp-fault-injection/src/injected_fault_message.rs create mode 100644 llama-cpp-fault-injection/src/lib.rs create mode 100644 llama-cpp-wrapper-sources/src/fault_injection_define.rs diff --git a/Cargo.lock b/Cargo.lock index 7bb2a2c17..3b8ec34cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1171,6 +1171,7 @@ dependencies = [ "llama-cpp-bindings-sys", "llama-cpp-bindings-types", "llama-cpp-error-recorder", + "llama-cpp-fault-injection", "llama-cpp-ffi-status", "llama-cpp-gbnf", "llama-cpp-log-decoder", @@ -1214,6 +1215,7 @@ dependencies = [ "llama-cpp-bindings", "llama-cpp-bindings-sys", "llama-cpp-bindings-types", + "llama-cpp-fault-injection", "llama-cpp-test-harness", "serde_json", "toktrie", @@ -1235,6 +1237,13 @@ dependencies = [ "log", ] +[[package]] +name = "llama-cpp-fault-injection" +version = "0.26.0" +dependencies = [ + "llama-cpp-bindings-sys", +] + [[package]] name = "llama-cpp-ffi-status" version = "0.26.0" diff --git a/Cargo.toml b/Cargo.toml index 1c2f6e953..5d9af5ca5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "llama-cpp-bindings", "llama-cpp-bindings-tests", "llama-cpp-error-recorder", + "llama-cpp-fault-injection", "llama-cpp-ffi-status", "llama-cpp-gbnf", "llama-cpp-log-decoder", @@ -43,6 +44,7 @@ llama-cpp-gbnf = { path = "llama-cpp-gbnf", version = "=0.26.0" } llama-cpp-log-decoder = { path = "llama-cpp-log-decoder", version = "=0.26.0" } llama-cpp-test-harness = { path = "llama-cpp-test-harness", version = "=0.26.0" } llama-cpp-test-harness-macros = { path = "llama-cpp-test-harness-macros", version = "=0.26.0" } +llama-cpp-fault-injection = { path = "llama-cpp-fault-injection" } llama-cpp-wrapper-error-fixture = { path = "llama-cpp-wrapper-error-fixture" } llama-cpp-wrapper-sources = { path = "llama-cpp-wrapper-sources", version = "=0.26.0" } llguidance = "=1.7.0" diff --git a/llama-cpp-bindings-build/Cargo.toml b/llama-cpp-bindings-build/Cargo.toml index 34039785d..f69f99f99 100644 --- a/llama-cpp-bindings-build/Cargo.toml +++ b/llama-cpp-bindings-build/Cargo.toml @@ -20,6 +20,7 @@ cuda-no-vmm = ["cuda"] metal = [] dynamic-link = [] dynamic-backends = ["dynamic-link"] +fault-injection = [] vulkan = [] openmp = [] rocm = [] diff --git a/llama-cpp-bindings-build/src/bindgen_config.rs b/llama-cpp-bindings-build/src/bindgen_config.rs index 40fd9dc67..e8da81c16 100644 --- a/llama-cpp-bindings-build/src/bindgen_config.rs +++ b/llama-cpp-bindings-build/src/bindgen_config.rs @@ -3,6 +3,8 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; +use llama_cpp_wrapper_sources::fault_injection_define::FAULT_INJECTION_DEFINE; + use crate::BuildError; use crate::android_ndk::AndroidNdk; use crate::debug_log; @@ -213,6 +215,10 @@ fn create_base_builder(llama_src: &Path, callbacks: BindingCallbacks) -> bindgen builder = builder.blocklist_function(function); } + if cfg!(feature = "fault-injection") { + builder = builder.clang_arg(format!("-D{FAULT_INJECTION_DEFINE}")); + } + builder } diff --git a/llama-cpp-bindings-build/src/cpp_wrapper.rs b/llama-cpp-bindings-build/src/cpp_wrapper.rs index d44afddcc..c0ea17f64 100644 --- a/llama-cpp-bindings-build/src/cpp_wrapper.rs +++ b/llama-cpp-bindings-build/src/cpp_wrapper.rs @@ -1,4 +1,5 @@ use llama_cpp_wrapper_sources::cpp_standard::CPP_STANDARD; +use llama_cpp_wrapper_sources::fault_injection_define::FAULT_INJECTION_DEFINE; use llama_cpp_wrapper_sources::vendored_include_dirs::VENDORED_INCLUDE_DIRS; use llama_cpp_wrapper_sources::wrapper_include_dirs::WRAPPER_INCLUDE_DIRS; use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; @@ -21,6 +22,10 @@ pub fn compile_cpp_wrappers(target_os: TargetOs) -> Result<(), BuildError> { build.flag_if_supported(format!("-std={CPP_STANDARD}")); + if cfg!(feature = "fault-injection") { + build.define(FAULT_INJECTION_DEFINE, None); + } + for source in WRAPPER_SOURCES { build.file(source); } diff --git a/llama-cpp-bindings-sys/Cargo.toml b/llama-cpp-bindings-sys/Cargo.toml index 29c9cfeea..4c2b20e94 100644 --- a/llama-cpp-bindings-sys/Cargo.toml +++ b/llama-cpp-bindings-sys/Cargo.toml @@ -78,6 +78,7 @@ cuda-no-vmm = ["cuda", "llama-cpp-bindings-build/cuda-no-vmm"] metal = ["llama-cpp-bindings-build/metal"] dynamic-link = ["llama-cpp-bindings-build/dynamic-link"] dynamic-backends = ["dynamic-link", "llama-cpp-bindings-build/dynamic-backends"] +fault-injection = ["llama-cpp-bindings-build/fault-injection"] vulkan = ["llama-cpp-bindings-build/vulkan"] openmp = ["llama-cpp-bindings-build/openmp"] rocm = ["llama-cpp-bindings-build/rocm"] diff --git a/llama-cpp-bindings-sys/wrapper.h b/llama-cpp-bindings-sys/wrapper.h index c9c12e8b1..e45b1fc1b 100644 --- a/llama-cpp-bindings-sys/wrapper.h +++ b/llama-cpp-bindings-sys/wrapper.h @@ -5,6 +5,7 @@ #include "wrapper_chat_templates.h" #include "wrapper_common.h" #include "wrapper_context.h" +#include "wrapper_fault_injection.h" #include "wrapper_fit.h" #include "wrapper_gbnf.h" #include "wrapper_mtmd.h" diff --git a/llama-cpp-bindings-sys/wrapper_batch.cpp b/llama-cpp-bindings-sys/wrapper_batch.cpp index ef7722234..6c2e51d61 100644 --- a/llama-cpp-bindings-sys/wrapper_batch.cpp +++ b/llama-cpp-bindings-sys/wrapper_batch.cpp @@ -1,4 +1,5 @@ #include "wrapper_batch.h" +#include "wrapper_fault_injection.h" #include "wrapper_report_current_exception.h" #include @@ -29,6 +30,7 @@ extern "C" auto llama_rs_batch_ext_init( } try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_BATCH_EXT_INIT_THROWS); *out_batch = llama_batch_ext_init(ctx); return LLAMA_RS_BATCH_EXT_INIT_OK; @@ -73,6 +75,7 @@ extern "C" auto llama_rs_batch_ext_add_entry( } try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_BATCH_EXT_ADD_ENTRY_THROWS); const gsl::span seq_id_span(seq_ids, n_seq_ids); const int32_t index = llama_batch_ext_add_token(batch, seq_id_span[0], token); diff --git a/llama-cpp-bindings-sys/wrapper_chat_session.cpp b/llama-cpp-bindings-sys/wrapper_chat_session.cpp index 406cbff71..10e5df6d9 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_session.cpp +++ b/llama-cpp-bindings-sys/wrapper_chat_session.cpp @@ -1,4 +1,5 @@ #include "wrapper_chat_session.h" +#include "wrapper_fault_injection.h" #include "wrapper_chat_templates.h" #include "wrapper_chat_templates_handle.h" #include "wrapper_parsed_chat.h" @@ -253,7 +254,10 @@ extern "C" auto llama_rs_chat_session_prompt_part_text( llama_rs_chat_session_handle session, size_t index, size_t * out_length) -> const char * { - return llama_rs_string_view(session->session.prompt().parts.at(index).text, out_length); + return llama_rs_text_view_unless_fault_injected( + LLAMA_RS_FAULT_POINT_CHAT_SESSION_PROMPT_PART_TEXT_NOT_UTF8, + session->session.prompt().parts.at(index).text, + out_length); } extern "C" auto llama_rs_chat_session_prompt_part_is_input(llama_rs_chat_session_handle session, size_t index) @@ -263,7 +267,8 @@ extern "C" auto llama_rs_chat_session_prompt_part_is_input(llama_rs_chat_session extern "C" auto llama_rs_chat_session_grammar(llama_rs_chat_session_handle session, size_t * out_length) -> const char * { - return llama_rs_string_view(session->session.grammar(), out_length); + return llama_rs_text_view_unless_fault_injected( + LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_NOT_UTF8, session->session.grammar(), out_length); } extern "C" auto llama_rs_chat_session_grammar_lazy(llama_rs_chat_session_handle session) -> bool { @@ -276,6 +281,10 @@ extern "C" auto llama_rs_chat_session_grammar_trigger_count(llama_rs_chat_sessio extern "C" auto llama_rs_chat_session_grammar_trigger_type(llama_rs_chat_session_handle session, size_t index) -> llama_rs_chat_grammar_trigger_type { + if (llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_TRIGGER_TYPE_UNRECOGNIZED)) { + return LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_UNRECOGNIZED; + } + switch (session->sampling.grammar_triggers.at(index).type) { case COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN: return LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_TOKEN; @@ -287,14 +296,17 @@ extern "C" auto llama_rs_chat_session_grammar_trigger_type(llama_rs_chat_session return LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL; } - return LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL; + return LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_UNRECOGNIZED; } extern "C" auto llama_rs_chat_session_grammar_trigger_value( llama_rs_chat_session_handle session, size_t index, size_t * out_length) -> const char * { - return llama_rs_string_view(session->sampling.grammar_triggers.at(index).value, out_length); + return llama_rs_text_view_unless_fault_injected( + LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_TRIGGER_VALUE_NOT_UTF8, + session->sampling.grammar_triggers.at(index).value, + out_length); } extern "C" auto llama_rs_chat_session_grammar_trigger_token(llama_rs_chat_session_handle session, size_t index) @@ -372,6 +384,8 @@ extern "C" auto llama_rs_chat_session_parse( } try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_CHAT_SESSION_PARSE_THROWS); + common_chat_parser_params parser_params; parser_params.format = session->session.format(); parser_params.generation_prompt = session->generation_prompt; @@ -432,6 +446,8 @@ extern "C" auto llama_rs_chat_session_classify( } try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_CHAT_SESSION_CLASSIFY_THROWS); + common_chat_input parse_input = chat_input_of(text, text_length, byte_tokens); parse_input.prepend(session->generation_prompt); @@ -523,6 +539,8 @@ extern "C" auto llama_rs_regex_escape( } try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_REGEX_ESCAPE_THROWS); + *out_string = llama_rs_dup_string(regex_escape(std::string(text, text_length))); if (*out_string == nullptr) { return LLAMA_RS_REGEX_ESCAPE_ERROR_STRING_ALLOCATION_FAILED; diff --git a/llama-cpp-bindings-sys/wrapper_chat_session.h b/llama-cpp-bindings-sys/wrapper_chat_session.h index 6ea75b3e3..3169dc1a1 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_session.h +++ b/llama-cpp-bindings-sys/wrapper_chat_session.h @@ -85,6 +85,7 @@ typedef enum llama_rs_chat_grammar_trigger_type { LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_WORD, LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN, LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL, + LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_UNRECOGNIZED, } llama_rs_chat_grammar_trigger_type; size_t llama_rs_chat_session_grammar_trigger_count(llama_rs_chat_session_handle session); diff --git a/llama-cpp-bindings-sys/wrapper_common.cpp b/llama-cpp-bindings-sys/wrapper_common.cpp index 50d355819..134694b4c 100644 --- a/llama-cpp-bindings-sys/wrapper_common.cpp +++ b/llama-cpp-bindings-sys/wrapper_common.cpp @@ -1,4 +1,5 @@ #include "wrapper_common.h" +#include "wrapper_fault_injection.h" #include #include @@ -93,7 +94,10 @@ extern "C" auto llama_rs_sampler_init_grammar( return LLAMA_RS_SAMPLER_INIT_GRAMMAR_NULL_OUT_ERROR_ARG; } try { - *out_sampler = llama_sampler_init_grammar(vocab, grammar_str, grammar_root); + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GRAMMAR_THROWS); + *out_sampler = llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GRAMMAR_RETURNS_NULL) + ? nullptr + : llama_sampler_init_grammar(vocab, grammar_str, grammar_root); if (*out_sampler == nullptr) { return LLAMA_RS_SAMPLER_INIT_GRAMMAR_LLAMA_CPP_RETURNED_NULL; } @@ -130,6 +134,7 @@ extern "C" auto llama_rs_sampler_init_grammar_lazy_patterns( return LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_NULL_OUT_ERROR_ARG; } try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_THROWS); *out_sampler = llama_sampler_init_grammar_lazy_patterns( vocab, grammar_str, @@ -367,6 +372,7 @@ extern "C" auto llama_rs_sampler_accept( return LLAMA_RS_SAMPLER_ACCEPT_NULL_OUT_ERROR_ARG; } try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_SAMPLER_ACCEPT_THROWS); llama_sampler_accept(sampler, token); return LLAMA_RS_SAMPLER_ACCEPT_OK; } catch (...) { @@ -561,3 +567,18 @@ extern "C" auto llama_rs_sampler_apply( LLAMA_RS_SAMPLER_APPLY_ERROR_STRING_ALLOCATION_FAILED); } } + +extern "C" auto llama_rs_token_to_piece( + const struct llama_vocab * vocab, + llama_token token, + char * buf, + int32_t length, + int32_t lstrip, + bool special) -> int32_t { + if (llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_TOKEN_TO_PIECE_FAILS) || + (special && llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_SPECIAL_TOKEN_TO_PIECE_FAILS))) { + return -(length + 1); + } + + return llama_token_to_piece(vocab, token, buf, length, lstrip, special); +} diff --git a/llama-cpp-bindings-sys/wrapper_common.h b/llama-cpp-bindings-sys/wrapper_common.h index 9bae6830e..a65763633 100644 --- a/llama-cpp-bindings-sys/wrapper_common.h +++ b/llama-cpp-bindings-sys/wrapper_common.h @@ -274,6 +274,14 @@ llama_rs_sampler_apply_status llama_rs_sampler_apply( struct llama_token_data_array * data_array, char ** out_error); +int32_t llama_rs_token_to_piece( + const struct llama_vocab * vocab, + llama_token token, + char * buf, + int32_t length, + int32_t lstrip, + bool special); + #ifdef __cplusplus } #endif diff --git a/llama-cpp-bindings-sys/wrapper_context.cpp b/llama-cpp-bindings-sys/wrapper_context.cpp index 0da34bc02..865deb5f1 100644 --- a/llama-cpp-bindings-sys/wrapper_context.cpp +++ b/llama-cpp-bindings-sys/wrapper_context.cpp @@ -1,5 +1,7 @@ #include "wrapper_context.h" +#include "wrapper_fault_injection.h" +#include #include #include @@ -81,3 +83,23 @@ extern "C" auto llama_rs_context_graph_produces_nextn_hidden_state( extern "C" auto llama_rs_model_causal_attn(const struct llama_model * model) -> bool { return model->hparams.causal_attn; } + +extern "C" auto llama_rs_get_logits_ith(struct llama_context * ctx, int32_t token_index) -> float * { + if (llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_GET_LOGITS_ITH_RETURNS_NULL)) { + return nullptr; + } + + return llama_get_logits_ith(ctx, token_index); +} + +extern "C" auto llama_rs_set_adapters_lora( + struct llama_context * ctx, + struct llama_adapter_lora ** adapters, + size_t n_adapters, + float * scales) -> int32_t { + if (llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_SET_ADAPTERS_LORA_FAILS)) { + return -1; + } + + return llama_set_adapters_lora(ctx, adapters, n_adapters, scales); +} diff --git a/llama-cpp-bindings-sys/wrapper_context.h b/llama-cpp-bindings-sys/wrapper_context.h index a382190d9..7a9b9030e 100644 --- a/llama-cpp-bindings-sys/wrapper_context.h +++ b/llama-cpp-bindings-sys/wrapper_context.h @@ -1,12 +1,14 @@ #pragma once #include +#include #include #ifdef __cplusplus extern "C" { #endif +struct llama_adapter_lora; struct llama_context; struct llama_model; @@ -38,6 +40,14 @@ llama_rs_context_nextn_hidden_state_probe_status llama_rs_context_graph_produces bool llama_rs_model_causal_attn(const struct llama_model * model); +float * llama_rs_get_logits_ith(struct llama_context * ctx, int32_t token_index); + +int32_t llama_rs_set_adapters_lora( + struct llama_context * ctx, + struct llama_adapter_lora ** adapters, + size_t n_adapters, + float * scales); + #ifdef __cplusplus } #endif diff --git a/llama-cpp-bindings-sys/wrapper_fault_injection.cpp b/llama-cpp-bindings-sys/wrapper_fault_injection.cpp new file mode 100644 index 000000000..184a9ba7a --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_fault_injection.cpp @@ -0,0 +1,82 @@ +#include "wrapper_fault_injection.h" +#include "wrapper_utils.h" + +#include +#include +#include + +#ifdef LLAMA_RS_FAULT_INJECTION +namespace { + +constexpr const char * INJECTED_FAULT_MESSAGE = "llama-cpp-bindings injected a fault"; + +struct armed_fault { + llama_rs_fault_point point = LLAMA_RS_FAULT_POINT_NONE; + size_t passing_hits = 0; +}; + +auto armed_fault_on_this_thread() -> armed_fault & { + thread_local armed_fault fault; + + return fault; +} + +} // namespace + +extern "C" void llama_rs_fault_injection_arm(llama_rs_fault_point point, size_t passing_hits) { + armed_fault_on_this_thread() = armed_fault{point, passing_hits}; +} + +extern "C" void llama_rs_fault_injection_disarm() { + armed_fault_on_this_thread() = armed_fault{}; +} + +extern "C" auto llama_rs_fault_injection_message() -> const char * { + return INJECTED_FAULT_MESSAGE; +} + +auto llama_rs_fault_injected(llama_rs_fault_point point) -> bool { + armed_fault & fault = armed_fault_on_this_thread(); + + if (fault.point != point) { + return false; + } + + if (fault.passing_hits > 0) { + fault.passing_hits -= 1; + + return false; + } + + return true; +} + +void llama_rs_throw_if_fault_injected(llama_rs_fault_point point) { + if (llama_rs_fault_injected(point)) { + throw std::runtime_error(INJECTED_FAULT_MESSAGE); + } +} +#else +auto llama_rs_fault_injected([[maybe_unused]] llama_rs_fault_point point) -> bool { + return false; +} + +void llama_rs_throw_if_fault_injected([[maybe_unused]] llama_rs_fault_point point) {} +#endif + +auto llama_rs_text_view_unless_fault_injected( + llama_rs_fault_point point, + const std::string & text, + size_t * out_length) -> const char * { + if (llama_rs_fault_injected(point)) { + static const std::string not_utf8 = "\xff"; + + return llama_rs_string_view(not_utf8, out_length); + } + + return llama_rs_string_view(text, out_length); +} + +extern "C" auto llama_rs_fault_point_reached(llama_rs_fault_point point) -> bool { + return llama_rs_fault_injected(point); +} diff --git a/llama-cpp-bindings-sys/wrapper_fault_injection.h b/llama-cpp-bindings-sys/wrapper_fault_injection.h new file mode 100644 index 000000000..4fd53f904 --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_fault_injection.h @@ -0,0 +1,68 @@ +#pragma once + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum llama_rs_fault_point { + LLAMA_RS_FAULT_POINT_NONE = 0, + LLAMA_RS_FAULT_POINT_BATCH_EXT_INIT_THROWS, + LLAMA_RS_FAULT_POINT_BATCH_EXT_ADD_ENTRY_THROWS, + LLAMA_RS_FAULT_POINT_CHAT_SESSION_CLASSIFY_THROWS, + LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_NOT_UTF8, + LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_TRIGGER_TYPE_UNRECOGNIZED, + LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_TRIGGER_VALUE_NOT_UTF8, + LLAMA_RS_FAULT_POINT_CHAT_SESSION_PARSE_THROWS, + LLAMA_RS_FAULT_POINT_CHAT_SESSION_PROMPT_PART_TEXT_NOT_UTF8, + LLAMA_RS_FAULT_POINT_GET_LOGITS_ITH_RETURNS_NULL, + LLAMA_RS_FAULT_POINT_LLGUIDANCE_FACTORY_FAILS, + LLAMA_RS_FAULT_POINT_MTMD_GET_MEMORY_USAGE_THROWS, + LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNK_TYPE_UNKNOWN, + LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNKS_GET_RETURNS_NULL, + LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNKS_INIT_THROWS, + LLAMA_RS_FAULT_POINT_MTMD_TOKENIZE_FROM_PARTS_THROWS, + LLAMA_RS_FAULT_POINT_PARSED_CHAT_CONTENT_NOT_UTF8, + LLAMA_RS_FAULT_POINT_PARSED_CHAT_REASONING_CONTENT_NOT_UTF8, + LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NOT_UTF8, + LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_ID_NOT_UTF8, + LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_NAME_NOT_UTF8, + LLAMA_RS_FAULT_POINT_REGEX_ESCAPE_THROWS, + LLAMA_RS_FAULT_POINT_SAMPLER_ACCEPT_THROWS, + LLAMA_RS_FAULT_POINT_SAMPLER_INIT_DIST_THROWS, + LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GRAMMAR_RETURNS_NULL, + LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GRAMMAR_THROWS, + LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_THROWS, + LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GREEDY_THROWS, + LLAMA_RS_FAULT_POINT_SET_ADAPTERS_LORA_FAILS, + LLAMA_RS_FAULT_POINT_SPECIAL_TOKEN_TO_PIECE_FAILS, + LLAMA_RS_FAULT_POINT_TOKEN_TO_PIECE_FAILS, +} llama_rs_fault_point; + +bool llama_rs_fault_point_reached(llama_rs_fault_point point); + +#ifdef LLAMA_RS_FAULT_INJECTION +void llama_rs_fault_injection_arm(llama_rs_fault_point point, size_t passing_hits); + +void llama_rs_fault_injection_disarm(void); + +const char * llama_rs_fault_injection_message(void); +#endif + +#ifdef __cplusplus +} + +#include +#include + +auto llama_rs_fault_injected(llama_rs_fault_point point) -> bool; + +void llama_rs_throw_if_fault_injected(llama_rs_fault_point point); + +auto llama_rs_text_view_unless_fault_injected( + llama_rs_fault_point point, + const std::string & text, + size_t * out_length) -> const char *; +#endif diff --git a/llama-cpp-bindings-sys/wrapper_mtmd.cpp b/llama-cpp-bindings-sys/wrapper_mtmd.cpp index 107e7713c..33c51a302 100644 --- a/llama-cpp-bindings-sys/wrapper_mtmd.cpp +++ b/llama-cpp-bindings-sys/wrapper_mtmd.cpp @@ -1,4 +1,5 @@ #include "wrapper_mtmd.h" +#include "wrapper_fault_injection.h" #include #include #include @@ -202,6 +203,8 @@ extern "C" auto llama_rs_mtmd_tokenize_from_parts( } try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_MTMD_TOKENIZE_FROM_PARTS_THROWS); + int32_t const result = mtmd_tokenize_from_parts(ctx, output, parts, num_parts, add_special); switch (result) { case 0: @@ -340,6 +343,8 @@ extern "C" auto llama_rs_mtmd_get_memory_usage( *out_device_memory_count = 0; try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_MTMD_GET_MEMORY_USAGE_THROWS); + const mtmd_memory_usage usage = mtmd_get_memory_usage(mmproj_path, ctx_params); if (usage.backend_mem_usage.size() > device_memory_capacity) { @@ -439,6 +444,8 @@ extern "C" auto llama_rs_mtmd_input_chunks_init( *out_chunks = nullptr; try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNKS_INIT_THROWS); + *out_chunks = mtmd_input_chunks_init(); return LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_OK; @@ -478,3 +485,21 @@ extern "C" auto llama_rs_mtmd_input_chunk_copy( LLAMA_RS_MTMD_INPUT_CHUNK_COPY_ERROR_STRING_ALLOCATION_FAILED); } } + +extern "C" auto llama_rs_mtmd_input_chunks_get(const struct mtmd_input_chunks * chunks, size_t index) + -> const struct mtmd_input_chunk * { + if (llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNKS_GET_RETURNS_NULL)) { + return nullptr; + } + + return mtmd_input_chunks_get(chunks, index); +} + +extern "C" auto llama_rs_mtmd_input_chunk_get_type(const struct mtmd_input_chunk * chunk) + -> mtmd_input_chunk_type { + if (llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNK_TYPE_UNKNOWN)) { + return MTMD_INPUT_CHUNK_TYPE_COUNT; + } + + return mtmd_input_chunk_get_type(chunk); +} diff --git a/llama-cpp-bindings-sys/wrapper_mtmd.h b/llama-cpp-bindings-sys/wrapper_mtmd.h index 09aa0ea18..b6804c880 100644 --- a/llama-cpp-bindings-sys/wrapper_mtmd.h +++ b/llama-cpp-bindings-sys/wrapper_mtmd.h @@ -235,6 +235,12 @@ llama_rs_mtmd_input_chunk_copy_status llama_rs_mtmd_input_chunk_copy( struct mtmd_input_chunk ** out_chunk, char ** out_error); +const struct mtmd_input_chunk * llama_rs_mtmd_input_chunks_get( + const struct mtmd_input_chunks * chunks, + size_t index); + +enum mtmd_input_chunk_type llama_rs_mtmd_input_chunk_get_type(const struct mtmd_input_chunk * chunk); + #ifdef __cplusplus } #endif diff --git a/llama-cpp-bindings-sys/wrapper_parsed_chat.cpp b/llama-cpp-bindings-sys/wrapper_parsed_chat.cpp index b0e87be3b..226768b36 100644 --- a/llama-cpp-bindings-sys/wrapper_parsed_chat.cpp +++ b/llama-cpp-bindings-sys/wrapper_parsed_chat.cpp @@ -1,6 +1,6 @@ #include "wrapper_parsed_chat.h" +#include "wrapper_fault_injection.h" #include "wrapper_parsed_chat_handle.h" -#include "wrapper_utils.h" #include #include @@ -11,13 +11,15 @@ extern "C" auto llama_rs_parsed_chat_free(llama_rs_parsed_chat_handle handle) -> extern "C" auto llama_rs_parsed_chat_content(llama_rs_parsed_chat_handle handle, size_t * out_length) -> const char * { - return llama_rs_string_view(handle->message.content, out_length); + return llama_rs_text_view_unless_fault_injected( + LLAMA_RS_FAULT_POINT_PARSED_CHAT_CONTENT_NOT_UTF8, handle->message.content, out_length); } extern "C" auto llama_rs_parsed_chat_reasoning_content( llama_rs_parsed_chat_handle handle, size_t * out_length) -> const char * { - return llama_rs_string_view(handle->message.reasoning_content, out_length); + return llama_rs_text_view_unless_fault_injected( + LLAMA_RS_FAULT_POINT_PARSED_CHAT_REASONING_CONTENT_NOT_UTF8, handle->message.reasoning_content, out_length); } extern "C" auto llama_rs_parsed_chat_tool_call_count(llama_rs_parsed_chat_handle handle) -> size_t { @@ -28,19 +30,22 @@ extern "C" auto llama_rs_parsed_chat_tool_call_id( llama_rs_parsed_chat_handle handle, size_t index, size_t * out_length) -> const char * { - return llama_rs_string_view(handle->message.tool_calls[index].id, out_length); + return llama_rs_text_view_unless_fault_injected( + LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_ID_NOT_UTF8, handle->message.tool_calls[index].id, out_length); } extern "C" auto llama_rs_parsed_chat_tool_call_name( llama_rs_parsed_chat_handle handle, size_t index, size_t * out_length) -> const char * { - return llama_rs_string_view(handle->message.tool_calls[index].name, out_length); + return llama_rs_text_view_unless_fault_injected( + LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_NAME_NOT_UTF8, handle->message.tool_calls[index].name, out_length); } extern "C" auto llama_rs_parsed_chat_tool_call_arguments( llama_rs_parsed_chat_handle handle, size_t index, size_t * out_length) -> const char * { - return llama_rs_string_view(handle->message.tool_calls[index].arguments, out_length); + return llama_rs_text_view_unless_fault_injected( + LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NOT_UTF8, handle->message.tool_calls[index].arguments, out_length); } diff --git a/llama-cpp-bindings-sys/wrapper_sampler_init.cpp b/llama-cpp-bindings-sys/wrapper_sampler_init.cpp index ecadf36bb..8ce8e4d13 100644 --- a/llama-cpp-bindings-sys/wrapper_sampler_init.cpp +++ b/llama-cpp-bindings-sys/wrapper_sampler_init.cpp @@ -1,4 +1,5 @@ #include "wrapper_sampler_init.h" +#include "wrapper_fault_injection.h" #include #include "wrapper_report_current_exception.h" @@ -61,14 +62,20 @@ extern "C" auto llama_rs_sampler_chain_init( extern "C" auto llama_rs_sampler_init_greedy( struct llama_sampler ** out_sampler, char ** out_error) -> llama_rs_sampler_init_status { - return init_sampler(out_sampler, out_error, []() -> struct llama_sampler * { return llama_sampler_init_greedy(); }); + return init_sampler(out_sampler, out_error, []() -> struct llama_sampler * { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GREEDY_THROWS); + return llama_sampler_init_greedy(); + }); } extern "C" auto llama_rs_sampler_init_dist( uint32_t seed, struct llama_sampler ** out_sampler, char ** out_error) -> llama_rs_sampler_init_status { - return init_sampler(out_sampler, out_error, [seed]() -> struct llama_sampler * { return llama_sampler_init_dist(seed); }); + return init_sampler(out_sampler, out_error, [seed]() -> struct llama_sampler * { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_SAMPLER_INIT_DIST_THROWS); + return llama_sampler_init_dist(seed); + }); } extern "C" auto llama_rs_sampler_init_top_k( diff --git a/llama-cpp-bindings-tests/Cargo.toml b/llama-cpp-bindings-tests/Cargo.toml index 1ec14b151..7a0156291 100644 --- a/llama-cpp-bindings-tests/Cargo.toml +++ b/llama-cpp-bindings-tests/Cargo.toml @@ -13,6 +13,7 @@ encoding_rs = { workspace = true } llama-cpp-bindings = { workspace = true } llama-cpp-bindings-sys = { workspace = true } llama-cpp-bindings-types = { workspace = true } +llama-cpp-fault-injection = { workspace = true } llama-cpp-test-harness = { workspace = true } serde_json = { workspace = true } toktrie = { workspace = true } diff --git a/llama-cpp-bindings-tests/src/chat_answer.rs b/llama-cpp-bindings-tests/src/chat_answer.rs index ebd7fd892..b5a80a26f 100644 --- a/llama-cpp-bindings-tests/src/chat_answer.rs +++ b/llama-cpp-bindings-tests/src/chat_answer.rs @@ -1,6 +1,3 @@ -use anyhow::Result; -use anyhow::bail; -use llama_cpp_bindings::ChatMessageParseOutcome; use llama_cpp_bindings::ChatOutputCompleteness; use llama_cpp_bindings::ChatSession; use llama_cpp_bindings::TokenUsage; @@ -10,26 +7,32 @@ use crate::recognized_answer::recognized_answer; const REASONING_PROBE: &str = "x"; -fn thinking_tags(session: &ChatSession<'_>) -> Result> { - let mut tags = session.thinking_end_tags()?; +fn thinking_tags(session: &ChatSession<'_>) -> Vec { + let mut tags = session + .thinking_end_tags() + .expect("the session's thinking end tags must be UTF-8"); - tags.push(session.thinking_start_tag()?); + tags.push( + session + .thinking_start_tag() + .expect("the session's thinking start tag must be UTF-8"), + ); - Ok(tags) + tags } -fn prompted_reasoning(session: &ChatSession<'_>) -> Result { - let probed = recognized_answer(session, REASONING_PROBE, ChatOutputCompleteness::Partial)?; +fn prompted_reasoning(session: &ChatSession<'_>) -> String { + let probed = recognized_answer(session, REASONING_PROBE, ChatOutputCompleteness::Partial); if probed.content == REASONING_PROBE && probed.reasoning_content.is_empty() { - return Ok(String::new()); + return String::new(); } - let Some(prompted) = probed.reasoning_content.strip_suffix(REASONING_PROBE) else { - bail!("the probe must continue either the prompt's reasoning or its content; {probed:?}"); - }; - - Ok(prompted.to_owned()) + probed + .reasoning_content + .strip_suffix(REASONING_PROBE) + .expect("the probe must continue either the prompt's reasoning or its content") + .to_owned() } /// A generated answer to a chat session, with what the classifier attributed to each section. @@ -43,14 +46,10 @@ impl ChatAnswer { /// Asserts that every generated token is content, and that no thinking tag of `session` /// reached the content stream. /// - /// # Errors - /// - /// Forwards errors of reading the session's thinking tags. - /// /// # Panics /// /// Panics when the answer reasons, holds a token of no section, or leaks a thinking tag. - pub fn assert_content_without_reasoning(&self, session: &ChatSession<'_>) -> Result<()> { + pub fn assert_content_without_reasoning(&self, session: &ChatSession<'_>) { let Self { outcome, usage } = self; assert_ne!(outcome.generated.text(), "", "{outcome:?}"); @@ -65,37 +64,31 @@ impl ChatAnswer { "{usage:?}" ); - for tag in thinking_tags(session)? { + for tag in thinking_tags(session) { assert!( !outcome.content_stream.contains(&tag), "the content stream leaked the thinking tag {tag:?}; {outcome:?}" ); } - - Ok(()) } /// Asserts that the answer reasons, and that the reasoning and content streams are exactly /// what the session's parser extracts, apart from the reasoning text the generation prompt /// itself holds, which the parser shows in front of the first generated text. /// - /// # Errors - /// - /// Forwards errors of parsing the answer or reading the session's thinking tags, and fails - /// when the parser does not recognize the answer. - /// /// # Panics /// - /// Panics when the answer does not reason, holds a token of no section, leaks a thinking - /// tag, or streams text the parser does not extract. - pub fn assert_reasoning_matches_parse(&self, session: &ChatSession<'_>) -> Result<()> { + /// Panics when the parser fails on or does not recognize the answer, or when the answer + /// does not reason, holds a token of no section, leaks a thinking tag, or streams text the + /// parser does not extract. + pub fn assert_reasoning_matches_parse(&self, session: &ChatSession<'_>) { let Self { outcome, usage } = self; - let ChatMessageParseOutcome::Recognized(parsed) = - session.parse(&outcome.generated, ChatOutputCompleteness::Finished)? - else { - bail!("the session's parser must recognize the generated answer; {outcome:?}"); - }; - let prompted_reasoning = prompted_reasoning(session)?; + let parsed = session + .parse(&outcome.generated, ChatOutputCompleteness::Finished) + .expect("the session's parser must parse the generated answer") + .into_recognized() + .expect("the session's parser must recognize the generated answer"); + let prompted_reasoning = prompted_reasoning(session); assert!(outcome.observed_reasoning > 0, "{outcome:?}"); assert_eq!(outcome.observed_undeterminable, 0, "{outcome:?}"); @@ -116,11 +109,9 @@ impl ChatAnswer { ); assert_eq!(outcome.content_stream, parsed.content); - for tag in thinking_tags(session)? { + for tag in thinking_tags(session) { assert!(!outcome.reasoning_stream.contains(&tag), "{outcome:?}"); assert!(!outcome.content_stream.contains(&tag), "{outcome:?}"); } - - Ok(()) } } diff --git a/llama-cpp-bindings-tests/src/chat_answer_generation.rs b/llama-cpp-bindings-tests/src/chat_answer_generation.rs index 65f9f01e6..74ee04c16 100644 --- a/llama-cpp-bindings-tests/src/chat_answer_generation.rs +++ b/llama-cpp-bindings-tests/src/chat_answer_generation.rs @@ -1,4 +1,3 @@ -use anyhow::Result; use llama_cpp_bindings::BareJsonToolCalls; use llama_cpp_bindings::ChatSession; use llama_cpp_bindings::SampledTokenClassifier; @@ -19,24 +18,34 @@ pub struct ChatAnswerGeneration<'borrow, 'model> { } impl ChatAnswerGeneration<'_, '_> { - /// # Errors + /// # Panics /// - /// Forwards context creation, prompt tokenization, decoding, and sampling errors. - pub fn run(self) -> Result { + /// Panics when creating the context, tokenizing or decoding the prompt, or generating the + /// answer fails. + #[must_use] + pub fn run(self) -> ChatAnswer { let Self { fixture, session, sampler, max_generated_tokens, } = self; - let mut context = fixture.build_context()?; - let prompt_tokens = session.tokenize_prompt()?; + let mut context = fixture + .build_context() + .expect("the fixture's context must build"); + let prompt_tokens = session + .tokenize_prompt() + .expect("the session's prompt must tokenize"); let mut classifier = SampledTokenClassifier::for_chat_session(session, BareJsonToolCalls::Detect); let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - context.decode(&batch)?; + classifier + .feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false) + .expect("the prompt must fit the batch"); + context + .decode(&batch) + .expect("decoding the prompt must succeed"); classifier.commit_prompt_tokens(); let initial_position = batch.n_tokens(); @@ -48,11 +57,11 @@ impl ChatAnswerGeneration<'_, '_> { initial_position, max_generated_tokens, } - .run()?; + .run(); - Ok(ChatAnswer { + ChatAnswer { outcome, usage: classifier.into_usage(), - }) + } } } diff --git a/llama-cpp-bindings-tests/src/chunk_token_breakdown.rs b/llama-cpp-bindings-tests/src/chunk_token_breakdown.rs index 8b07191e8..f9aaabdcf 100644 --- a/llama-cpp-bindings-tests/src/chunk_token_breakdown.rs +++ b/llama-cpp-bindings-tests/src/chunk_token_breakdown.rs @@ -1,5 +1,3 @@ -use anyhow::Context; -use anyhow::Result; use llama_cpp_bindings::mtmd::MtmdInputChunkType; use llama_cpp_bindings::mtmd::MtmdInputChunks; @@ -10,27 +8,34 @@ pub struct ChunkTokenBreakdown { } impl ChunkTokenBreakdown { - /// # Errors + /// # Panics /// - /// Forwards chunk access and chunk-type classification errors. - pub fn from_chunks(chunks: &MtmdInputChunks) -> Result { + /// Panics when a chunk below the chunk count is missing or reports a type unknown to the + /// bindings. + #[must_use] + pub fn from_chunks(chunks: &MtmdInputChunks) -> Self { let mut breakdown = Self { text: 0, image: 0, audio: 0, }; + for index in 0..chunks.len() { let chunk = chunks .get(index) - .with_context(|| format!("chunk index {index} is missing"))?; - let n_tokens = u64::try_from(chunk.n_tokens())?; - match chunk.chunk_type()? { + .expect("every index below the chunk count must hold a chunk"); + let n_tokens = u64::try_from(chunk.n_tokens()).expect("a chunk's token count fits u64"); + + match chunk + .chunk_type() + .expect("every chunk must have a known type") + { MtmdInputChunkType::Text => breakdown.text += n_tokens, MtmdInputChunkType::Image => breakdown.image += n_tokens, MtmdInputChunkType::Audio => breakdown.audio += n_tokens, } } - Ok(breakdown) + breakdown } } diff --git a/llama-cpp-bindings-tests/src/classify_sample_loop.rs b/llama-cpp-bindings-tests/src/classify_sample_loop.rs index 3ce598942..3331ed916 100644 --- a/llama-cpp-bindings-tests/src/classify_sample_loop.rs +++ b/llama-cpp-bindings-tests/src/classify_sample_loop.rs @@ -1,4 +1,3 @@ -use anyhow::Result; use llama_cpp_bindings::GenerationProgress; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::llama_batch::LlamaBatch; @@ -18,23 +17,29 @@ pub struct ClassifySampleLoop<'borrow, 'session, 'model> { } impl ClassifySampleLoop<'_, '_, '_> { - /// # Errors - /// Forwards [`SampledTokenClassifier::sample`] / [`SampledTokenClassifier::finish`] / - /// [`LlamaContext::decode`] / [`LlamaBatch::add`] errors verbatim. Stops on the end of generation, on - /// `max_generated_tokens` exhaustion, or on the first error. - pub fn run(self) -> Result { + /// Samples and classifies tokens until the end of generation or until + /// `max_generated_tokens` are generated. + /// + /// # Panics + /// + /// Panics when sampling, classifying, batching, or decoding a token fails. + #[must_use] + pub fn run(self) -> ClassifySampleLoopOutcome { let mut outcome = ClassifySampleLoopOutcome::default(); let mut ingest_outcomes = Vec::new(); let mut position = self.initial_position; let max_position = position + self.max_generated_tokens; while position < max_position { - let sampled = self.classifier.sample( - self.sampler, - self.context, - self.batch.n_tokens() - 1, - &mut ingest_outcomes, - )?; + let sampled = self + .classifier + .sample( + self.sampler, + self.context, + self.batch.n_tokens() - 1, + &mut ingest_outcomes, + ) + .expect("sampling and classifying the next token must succeed"); if sampled.progress == GenerationProgress::Ended { outcome.record_end_of_generation(self.classifier.current_section()); @@ -44,18 +49,23 @@ impl ClassifySampleLoop<'_, '_, '_> { self.batch.clear(); self.batch - .add(&SampledToken::Content(sampled.token), position, &[0], true)?; + .add(&SampledToken::Content(sampled.token), position, &[0], true) + .expect("the sampled token must fit the batch"); position += 1; - self.context.decode(self.batch)?; + self.context + .decode(self.batch) + .expect("decoding the sampled token must succeed"); } - self.classifier.finish(&mut ingest_outcomes)?; + self.classifier + .finish(&mut ingest_outcomes) + .expect("finishing the classification must succeed"); for ingest_outcome in &ingest_outcomes { outcome.record_outcome(ingest_outcome); } - Ok(outcome) + outcome } } diff --git a/llama-cpp-bindings-tests/src/classify_sample_loop_outcome.rs b/llama-cpp-bindings-tests/src/classify_sample_loop_outcome.rs index 33101d91d..39c59f91d 100644 --- a/llama-cpp-bindings-tests/src/classify_sample_loop_outcome.rs +++ b/llama-cpp-bindings-tests/src/classify_sample_loop_outcome.rs @@ -99,11 +99,34 @@ mod tests { #[test] fn counts_the_end_of_generation_in_the_section_it_ended() { - let mut outcome = ClassifySampleLoopOutcome::default(); - - outcome.record_end_of_generation(SampledTokenSection::Content); + let counts = [ + SampledTokenSection::Content, + SampledTokenSection::Reasoning, + SampledTokenSection::ToolCall, + SampledTokenSection::Pending, + ] + .map(|section| { + let mut outcome = ClassifySampleLoopOutcome::default(); + + outcome.record_end_of_generation(section); + + [ + u64::from(outcome.eog_seen), + outcome.observed_content, + outcome.observed_reasoning, + outcome.observed_tool_call, + outcome.observed_undeterminable, + ] + }); - assert!(outcome.eog_seen); - assert_eq!(outcome.observed_content, 1); + assert_eq!( + counts, + [ + [1, 1, 0, 0, 0], + [1, 0, 1, 0, 0], + [1, 0, 0, 1, 0], + [1, 0, 0, 0, 1], + ] + ); } } diff --git a/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs b/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs index 265b678d0..d49c7f5e8 100644 --- a/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs +++ b/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs @@ -1,4 +1,3 @@ -use anyhow::Result; use llama_cpp_bindings::ChatAssistantPrefill; use llama_cpp_bindings::ChatAssistantTurn; use llama_cpp_bindings::ChatSession; @@ -15,20 +14,21 @@ const PREFILLED_CONTENT: &str = "\n\n"; /// The prefill closes a reasoning block before the content begins, so the model writes only /// content even when its template cannot disable thinking. /// -/// # Errors +/// # Panics /// -/// Forwards message construction and chat session errors. +/// Panics when the question holds a NUL byte or the model's template cannot serve the request. +#[must_use] pub fn closed_reasoning_chat_session<'model>( model: &'model LlamaModel, question: &str, tools_json: &str, -) -> Result> { - Ok(ChatSession::new( +) -> ChatSession<'model> { + ChatSession::new( model, - &[LlamaChatMessage::new( - "user".to_owned(), - question.to_owned(), - )?], + &[ + LlamaChatMessage::new("user".to_owned(), question.to_owned()) + .expect("the question must hold no NUL byte"), + ], tools_json, &ChatSessionOptions { assistant_turn: ChatAssistantTurn::ContinueContent(ChatAssistantPrefill { @@ -39,5 +39,6 @@ pub fn closed_reasoning_chat_session<'model>( parallel_tool_calls: false, tool_choice: ChatToolChoice::Auto, }, - )?) + ) + .expect("the model's template must serve a prefilled answer") } diff --git a/llama-cpp-bindings-tests/src/ingested_generation.rs b/llama-cpp-bindings-tests/src/ingested_generation.rs index a4fea75e2..87d8781de 100644 --- a/llama-cpp-bindings-tests/src/ingested_generation.rs +++ b/llama-cpp-bindings-tests/src/ingested_generation.rs @@ -1,5 +1,3 @@ -use anyhow::Context as _; -use anyhow::Result; use llama_cpp_bindings::GenerationProgress; use llama_cpp_bindings::SampledTokenClassifier; use llama_cpp_bindings::ingest_outcome::IngestOutcome; @@ -10,25 +8,24 @@ use llama_cpp_bindings::model::ParseSpecialTokens; /// Feeds the tokens of `generated_text` to `classifier` as if the model generated them, then /// ends the generation with the model's end-of-sequence token. /// -/// # Errors -/// -/// Forwards tokenization and classification errors. -/// /// # Panics /// -/// Panics when the classifier ends the generation before the end-of-sequence token, or does -/// not end it on that token. +/// Panics when tokenizing or classifying fails, or when the classifier ends the generation +/// before the end-of-sequence token, or does not end it on that token. pub fn ingested_generation( classifier: &mut SampledTokenClassifier<'_, '_>, model: &LlamaModel, generated_text: &str, -) -> Result> { +) -> Vec { let mut outcomes = Vec::new(); - for token in model.str_to_token(generated_text, AddBos::Never, ParseSpecialTokens::Always)? { + for token in model + .str_to_token(generated_text, AddBos::Never, ParseSpecialTokens::Always) + .expect("the generated text must tokenize") + { assert_eq!( - classifier.ingest(token, &mut outcomes)?, - GenerationProgress::Continues + classifier.ingest(token, &mut outcomes), + Ok(GenerationProgress::Continues) ); } @@ -36,11 +33,11 @@ pub fn ingested_generation( classifier.ingest( model .token_eos() - .context("the model declares an EOS token")?, + .expect("the model must declare an EOS token"), &mut outcomes - )?, - GenerationProgress::Ended + ), + Ok(GenerationProgress::Ended) ); - Ok(outcomes) + outcomes } diff --git a/llama-cpp-bindings-tests/src/lib.rs b/llama-cpp-bindings-tests/src/lib.rs index 73ef8ce85..727c6acef 100644 --- a/llama-cpp-bindings-tests/src/lib.rs +++ b/llama-cpp-bindings-tests/src/lib.rs @@ -1,7 +1,4 @@ -#![cfg_attr( - not(test), - deny(clippy::unwrap_used, clippy::expect_used, clippy::panic) -)] +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::panic))] pub mod chat_answer; pub mod chat_answer_generation; pub mod chunk_token_breakdown; @@ -20,3 +17,4 @@ pub mod prime_kv_cache_with; pub mod prompt_question; pub mod recognized_answer; pub mod user_turn_chat_session; +pub mod weather_chat_session; diff --git a/llama-cpp-bindings-tests/src/media_question_chat_session.rs b/llama-cpp-bindings-tests/src/media_question_chat_session.rs index 951bb2c15..6ebfd31fd 100644 --- a/llama-cpp-bindings-tests/src/media_question_chat_session.rs +++ b/llama-cpp-bindings-tests/src/media_question_chat_session.rs @@ -11,13 +11,20 @@ use crate::user_turn_chat_session::user_turn_chat_session; /// # Errors /// /// Forwards message construction and chat session errors. +/// +/// # Panics +/// +/// Panics when llama.cpp's media marker is not UTF-8. pub fn media_question_chat_session<'model>( model: &'model LlamaModel, question: &str, ) -> Result> { user_turn_chat_session( model, - &format!("{}{question}", mtmd_default_marker()?), + &format!( + "{}{question}", + mtmd_default_marker().expect("llama.cpp's media marker must be UTF-8") + ), "[]", true, ) diff --git a/llama-cpp-bindings-tests/src/penalized_nucleus_sampler.rs b/llama-cpp-bindings-tests/src/penalized_nucleus_sampler.rs index aef70ae1f..d0581c09b 100644 --- a/llama-cpp-bindings-tests/src/penalized_nucleus_sampler.rs +++ b/llama-cpp-bindings-tests/src/penalized_nucleus_sampler.rs @@ -1,4 +1,3 @@ -use anyhow::Result; use llama_cpp_bindings::model::LlamaModel; use llama_cpp_bindings::sampling::LlamaSampler; @@ -7,16 +6,19 @@ const SAMPLING_SEED: u32 = 0x00C0_FFEE; /// A seeded sampler that penalizes repetition and samples from the nucleus, so a small model /// does not loop while its answer stays reproducible. /// -/// # Errors +/// # Panics /// -/// Forwards sampler construction errors. -pub fn penalized_nucleus_sampler(model: &LlamaModel) -> Result { - Ok(LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(SAMPLING_SEED)?, - ])?) +/// Panics when llama.cpp cannot build one of the samplers. +#[must_use] +pub fn penalized_nucleus_sampler(model: &LlamaModel) -> LlamaSampler { + LlamaSampler::chain_simple([ + LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0) + .expect("the penalties sampler must build"), + LlamaSampler::top_k(40).expect("the top-k sampler must build"), + LlamaSampler::top_p(0.9, 1).expect("the top-p sampler must build"), + LlamaSampler::min_p(0.05, 1).expect("the min-p sampler must build"), + LlamaSampler::temp(0.7).expect("the temperature sampler must build"), + LlamaSampler::dist(SAMPLING_SEED).expect("the distribution sampler must build"), + ]) + .expect("the sampler chain must build") } diff --git a/llama-cpp-bindings-tests/src/recognized_answer.rs b/llama-cpp-bindings-tests/src/recognized_answer.rs index 4b779bf9f..f7acdac31 100644 --- a/llama-cpp-bindings-tests/src/recognized_answer.rs +++ b/llama-cpp-bindings-tests/src/recognized_answer.rs @@ -1,25 +1,22 @@ -use anyhow::Result; -use anyhow::bail; -use llama_cpp_bindings::ChatMessageParseOutcome; use llama_cpp_bindings::ChatOutput; use llama_cpp_bindings::ChatOutputCompleteness; use llama_cpp_bindings::ChatSession; use llama_cpp_bindings::ParsedChatMessage; -/// Parses `answer` with the session's parser, failing when the parser does not recognize it. +/// Parses `answer` with the session's parser. /// -/// # Errors +/// # Panics /// -/// Forwards parse errors, and fails when the answer does not match the session's format. +/// Panics when parsing fails or the parser does not recognize the answer. +#[must_use] pub fn recognized_answer( session: &ChatSession<'_>, answer: &str, completeness: ChatOutputCompleteness, -) -> Result { - match session.parse(&ChatOutput::from(answer), completeness)? { - ChatMessageParseOutcome::Recognized(parsed) => Ok(parsed), - ChatMessageParseOutcome::Unrecognized(raw) => { - bail!("the session's parser must recognize {answer:?}; got {raw:?}") - } - } +) -> ParsedChatMessage { + session + .parse(&ChatOutput::from(answer), completeness) + .expect("the session's parser must parse the answer") + .into_recognized() + .expect("the session's parser must recognize the answer") } diff --git a/llama-cpp-bindings-tests/src/weather_chat_session.rs b/llama-cpp-bindings-tests/src/weather_chat_session.rs new file mode 100644 index 000000000..33ae98d5d --- /dev/null +++ b/llama-cpp-bindings-tests/src/weather_chat_session.rs @@ -0,0 +1,34 @@ +use llama_cpp_bindings::ChatAssistantTurn; +use llama_cpp_bindings::ChatSession; +use llama_cpp_bindings::ChatSessionOptions; +use llama_cpp_bindings::ChatToolChoice; +use llama_cpp_bindings::model::LlamaChatMessage; +use llama_cpp_bindings::model::LlamaModel; + +use crate::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; + +/// A chat session that asks for the weather in Paris, offering the weather tool with +/// `tool_choice`. +/// +/// # Panics +/// +/// Panics when the model's template cannot serve the request. +#[must_use] +pub fn weather_chat_session(model: &LlamaModel, tool_choice: ChatToolChoice) -> ChatSession<'_> { + ChatSession::new( + model, + &[LlamaChatMessage::new( + "user".to_owned(), + "What is the weather in Paris?".to_owned(), + ) + .expect("the question holds no NUL byte")], + GET_WEATHER_TOOLS_JSON, + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::New, + enable_thinking: false, + parallel_tool_calls: false, + tool_choice, + }, + ) + .expect("the model's template must serve the weather request") +} diff --git a/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs b/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs index 4c0764579..6f350c7ac 100644 --- a/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs +++ b/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs @@ -3,9 +3,13 @@ use anyhow::bail; use llama_cpp_bindings::ChatMessageParseOutcome; use llama_cpp_bindings::ChatOutput; use llama_cpp_bindings::ChatOutputCompleteness; +use llama_cpp_bindings::ChatToolChoice; +use llama_cpp_bindings::ParseChatMessageError; use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_bindings_tests::weather_chat_session::weather_chat_session; use llama_cpp_bindings_types::ParsedToolCall; +use llama_cpp_fault_injection::ArmedFault; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -42,7 +46,7 @@ use llama_cpp_test_harness::llama_test; n_ubatch = 64, )] fn parses_pure_content_response(fixture: &LlamaFixture<'_>) -> Result<()> { - let outcome = closed_reasoning_chat_session(fixture.model, "hello", "[]")?.parse( + let outcome = closed_reasoning_chat_session(fixture.model, "hello", "[]").parse( &ChatOutput::from("hello world"), ChatOutputCompleteness::Finished, )?; @@ -197,3 +201,39 @@ fn qwen35_chat_session_parses_every_answer_it_is_given(fixture: &LlamaFixture<'_ Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn every_parsed_field_refuses_text_that_is_not_utf8(fixture: &LlamaFixture<'_>) { + let session = weather_chat_session(fixture.model, ChatToolChoice::Auto); + let answer = ChatOutput::from( + "\n\n\nParis\n\n\n", + ); + let not_utf8 = String::from_utf8(vec![0xff]) + .expect_err("a lone 0xff byte is not UTF-8") + .utf8_error(); + let fault_points = [ + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_PARSED_CHAT_CONTENT_NOT_UTF8, + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_PARSED_CHAT_REASONING_CONTENT_NOT_UTF8, + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_ID_NOT_UTF8, + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_NAME_NOT_UTF8, + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NOT_UTF8, + ]; + + assert_eq!( + fault_points.map(|fault_point| { + let _fault = ArmedFault::at(fault_point); + + session + .parse(&answer, ChatOutputCompleteness::Finished) + .err() + }), + fault_points.map(|_fault_point| Some(ParseChatMessageError::ParsedTextNotUtf8(not_utf8))) + ); +} diff --git a/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs b/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs new file mode 100644 index 000000000..f2ab693ba --- /dev/null +++ b/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs @@ -0,0 +1,254 @@ +use std::str::Utf8Error; + +use llama_cpp_bindings::ChatOutput; +use llama_cpp_bindings::ChatOutputCompleteness; +use llama_cpp_bindings::ChatSessionError; +use llama_cpp_bindings::ChatTemplatesError; +use llama_cpp_bindings::ChatToolChoice; +use llama_cpp_bindings::FfiStatusError; +use llama_cpp_bindings::GrammarError; +use llama_cpp_bindings::ParseChatMessageError; +use llama_cpp_bindings::SamplerAcceptError; +use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_bindings_tests::weather_chat_session::weather_chat_session; +use llama_cpp_fault_injection::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message; +use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_test; + +fn not_utf8() -> Utf8Error { + String::from_utf8(vec![0xff]) + .expect_err("a lone 0xff byte is not UTF-8") + .utf8_error() +} + +#[llama_test( + model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, +)] +fn a_chat_session_needs_a_model_with_a_chat_template(fixture: &LlamaFixture<'_>) { + assert_eq!( + user_turn_chat_session(fixture.model, "hello", "[]", false) + .err() + .map(|failure| failure.downcast::().ok()), + Some(Some(ChatSessionError::Templates( + ChatTemplatesError::ModelHasNoChatTemplate + ))) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn every_prompt_reader_refuses_prompt_text_that_is_not_utf8(fixture: &LlamaFixture<'_>) { + let session = weather_chat_session(fixture.model, ChatToolChoice::Auto); + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_PROMPT_PART_TEXT_NOT_UTF8, + ); + + assert_eq!( + [ + session.prompt_parts().err(), + session.prompt().err(), + session.tokenize_prompt().err(), + ], + [(); 3].map(|()| Some(ChatSessionError::SessionTextNotUtf8(not_utf8()))) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn grammar_triggers_refuse_a_trigger_value_that_is_not_utf8(fixture: &LlamaFixture<'_>) { + let session = weather_chat_session(fixture.model, ChatToolChoice::Auto); + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_TRIGGER_VALUE_NOT_UTF8, + ); + + assert_eq!( + [ + session.grammar_triggers().err(), + session.grammar_sampler().err(), + ], + [(); 2].map(|()| Some(ChatSessionError::SessionTextNotUtf8(not_utf8()))) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn grammar_triggers_refuse_a_trigger_type_the_bindings_do_not_recognize( + fixture: &LlamaFixture<'_>, +) { + let session = weather_chat_session(fixture.model, ChatToolChoice::Auto); + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_TRIGGER_TYPE_UNRECOGNIZED, + ); + + assert_eq!( + session.grammar_triggers().err(), + Some(ChatSessionError::FfiStatus(FfiStatusError { + operation: "llama_rs_chat_session_grammar_trigger_type", + code: i64::from( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_UNRECOGNIZED + ), + })) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn the_grammar_sampler_refuses_a_grammar_that_is_not_utf8(fixture: &LlamaFixture<'_>) { + let session = weather_chat_session(fixture.model, ChatToolChoice::Required); + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_NOT_UTF8); + + assert_eq!( + session.grammar_sampler().err(), + Some(ChatSessionError::SessionTextNotUtf8(not_utf8())) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn the_grammar_sampler_reports_a_grammar_llama_cpp_cannot_load(fixture: &LlamaFixture<'_>) { + let session = weather_chat_session(fixture.model, ChatToolChoice::Required); + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GRAMMAR_THROWS); + + assert!(!session.grammar_is_lazy()); + assert_eq!( + session.grammar_sampler().err(), + Some(ChatSessionError::Grammar(GrammarError::Reported { + message: injected_fault_message(), + })) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn the_grammar_sampler_reports_a_generation_prompt_token_the_grammar_cannot_accept( + fixture: &LlamaFixture<'_>, +) { + let session = weather_chat_session(fixture.model, ChatToolChoice::Required); + let _fault = ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_ACCEPT_THROWS); + + assert_ne!(session.generation_prompt_tokens(), []); + assert_eq!( + session.grammar_sampler().err(), + Some(ChatSessionError::GrammarPrefill( + SamplerAcceptError::GrammarStateCorrupted { + message: injected_fault_message(), + } + )) + ); +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen2.5-7B-Instruct-GGUF", "qwen2.5-7b-instruct-q2_k.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn the_grammar_sampler_reports_a_trigger_word_llama_cpp_cannot_escape(fixture: &LlamaFixture<'_>) { + let session = weather_chat_session(fixture.model, ChatToolChoice::Auto); + let _fault = ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_REGEX_ESCAPE_THROWS); + + assert!(session.grammar_is_lazy()); + assert_eq!( + session.grammar_sampler().err(), + Some(ChatSessionError::Grammar(GrammarError::Reported { + message: injected_fault_message(), + })) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn the_grammar_sampler_reports_a_lazy_grammar_llama_cpp_cannot_load(fixture: &LlamaFixture<'_>) { + let session = weather_chat_session(fixture.model, ChatToolChoice::Auto); + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_THROWS, + ); + + assert_eq!( + session.grammar_sampler().err(), + Some(ChatSessionError::Grammar(GrammarError::Reported { + message: injected_fault_message(), + })) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn parsing_a_duck_typed_tool_call_reports_a_parser_llama_cpp_fails(fixture: &LlamaFixture<'_>) { + let session = weather_chat_session(fixture.model, ChatToolChoice::Auto); + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_PARSE_THROWS); + + assert!(!session.supports_tool_calls()); + assert_eq!( + session + .parse( + &ChatOutput::from( + "Checking.\n\n\nParis\n\n\n" + ), + ChatOutputCompleteness::Finished, + ) + .err(), + Some(ParseChatMessageError::Reported { + message: injected_fault_message(), + }) + ); +} diff --git a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs index 0407054a3..c8c6e92e1 100644 --- a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs +++ b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs @@ -110,7 +110,7 @@ fn raw_prompt_completion_usage_matches_the_classified_tokens( initial_position, max_generated_tokens, } - .run()?; + .run(); let t_main_end = ggml_time_us(); let duration = Duration::from_micros(u64::try_from(t_main_end - t_main_start)?); let total_observed = @@ -250,7 +250,7 @@ fn chat_completion_usage_matches_the_classified_tokens(fixture: &LlamaFixture<'_ initial_position, max_generated_tokens: 512, } - .run()?; + .run(); println!(); diff --git a/llama-cpp-bindings-tests/tests/context_creation_limits.rs b/llama-cpp-bindings-tests/tests/context_creation_limits.rs index cd5ff0c6f..0cd4b4b00 100644 --- a/llama-cpp-bindings-tests/tests/context_creation_limits.rs +++ b/llama-cpp-bindings-tests/tests/context_creation_limits.rs @@ -2,6 +2,8 @@ use std::num::NonZeroU32; use llama_cpp_bindings::LlamaContextLoadError; use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_fault_injection::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -95,3 +97,27 @@ fn new_context_rejects_more_sequences_than_a_causal_context_can_output(fixture: }) ); } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn creating_a_context_reports_a_batch_llama_cpp_cannot_allocate(fixture: &LlamaFixture<'_>) { + let _fault = ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_BATCH_EXT_INIT_THROWS); + + assert_eq!( + LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + ) + .err(), + Some(LlamaContextLoadError::Reported { + message: injected_fault_message(), + }) + ); +} diff --git a/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs b/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs index 7b14c8328..6fe83851f 100644 --- a/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs +++ b/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs @@ -3,9 +3,12 @@ use llama_cpp_bindings::DecodeError; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::context::params::LlamaAttentionType; +use llama_cpp_bindings::error::batch_transfer_error::BatchTransferError; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::AddBos; use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_fault_injection::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -102,3 +105,33 @@ fn decoding_more_tokens_than_the_micro_batch_on_an_encoder_returns_an_error( Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn decoding_reports_a_token_llama_cpp_cannot_add_to_its_batch( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_BATCH_EXT_ADD_ENTRY_THROWS); + + assert_eq!( + context.decode(&batch), + Err(DecodeError::BatchTransfer(BatchTransferError::Reported { + message: injected_fault_message(), + })) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs b/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs index 674605687..7472cd814 100644 --- a/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs +++ b/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs @@ -17,7 +17,7 @@ fn assert_duck_types_paris_weather_call(fixture: &LlamaFixture<'_>, answer: &str assert!(!session.supports_tool_calls()); assert_eq!( - recognized_answer(&session, answer, ChatOutputCompleteness::Finished)?.tool_calls, + recognized_answer(&session, answer, ChatOutputCompleteness::Finished).tool_calls, [paris_weather_tool_call()] ); diff --git a/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs b/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs index c9633a82c..f3fb90f85 100644 --- a/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs +++ b/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs @@ -1,9 +1,12 @@ use anyhow::Result; use llama_cpp_bindings::EncodeError; use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::error::batch_transfer_error::BatchTransferError; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::AddBos; use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_fault_injection::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -39,3 +42,34 @@ fn encoding_more_tokens_than_the_micro_batch_returns_an_error( Ok(()) } + +#[llama_test( + model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 64, + embeddings = true, +)] +fn encoding_reports_a_token_llama_cpp_cannot_add_to_its_batch( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_BATCH_EXT_ADD_ENTRY_THROWS); + + assert_eq!( + context.encode(&batch), + Err(EncodeError::BatchTransfer(BatchTransferError::Reported { + message: injected_fault_message(), + })) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/generation_end_classification.rs b/llama-cpp-bindings-tests/tests/generation_end_classification.rs index d355271e5..19ea87c25 100644 --- a/llama-cpp-bindings-tests/tests/generation_end_classification.rs +++ b/llama-cpp-bindings-tests/tests/generation_end_classification.rs @@ -39,7 +39,7 @@ fn qwen35_classifier_ends_generation_without_emitting_the_end_of_generation_toke &mut classifier, fixture.model, GENERATED_TEXT - )?), + )), GENERATED_TEXT ); @@ -62,7 +62,7 @@ fn gemma3_releases_a_held_bare_json_prefix_as_content_when_generation_ends( let session = user_turn_chat_session(fixture.model, "What is 2 + 2?", "[]", false)?; let mut classifier = SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); - let outcomes = ingested_generation(&mut classifier, fixture.model, GENERATED_TEXT)?; + let outcomes = ingested_generation(&mut classifier, fixture.model, GENERATED_TEXT); assert!(!session.supports_tool_calls()); assert_eq!(visible_text(&outcomes), GENERATED_TEXT); @@ -92,7 +92,7 @@ fn gemma3_attributes_a_bare_json_tool_call_object_to_the_tool_call( let session = user_turn_chat_session(fixture.model, "What is 2 + 2?", "[]", false)?; let mut classifier = SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); - let outcomes = ingested_generation(&mut classifier, fixture.model, GENERATED_TEXT)?; + let outcomes = ingested_generation(&mut classifier, fixture.model, GENERATED_TEXT); assert!( outcomes diff --git a/llama-cpp-bindings-tests/tests/grammar_sampler.rs b/llama-cpp-bindings-tests/tests/grammar_sampler.rs index 89bf0b90f..c5b2cf3f6 100644 --- a/llama-cpp-bindings-tests/tests/grammar_sampler.rs +++ b/llama-cpp-bindings-tests/tests/grammar_sampler.rs @@ -16,6 +16,7 @@ use llama_cpp_bindings::model::ParseSpecialTokens; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; +use llama_cpp_fault_injection::ArmedFault; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -155,7 +156,7 @@ fn sample_with_grammar_produces_constrained_output_in_loop( (*fixture.context_params).into_llama_context_params(), )?; - let session = closed_reasoning_chat_session(model, "Is the sky blue? yes or no", "[]")?; + let session = closed_reasoning_chat_session(model, "Is the sky blue? yes or no", "[]"); let tokens = session.tokenize_prompt()?; let mut batch = LlamaBatch::new(ContextLimits::of(&context)); @@ -181,7 +182,7 @@ fn sample_with_grammar_produces_constrained_output_in_loop( initial_position, max_generated_tokens: 10, } - .run()?; + .run(); let lowercase = outcome.generated.text().to_lowercase(); assert!( @@ -310,3 +311,22 @@ fn a_grammar_with_a_nul_byte_is_refused(fixture: &LlamaFixture<'_>) { .map(GrammarError::GrammarContainsNul) ); } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn a_valid_grammar_llama_cpp_refuses_to_load_is_reported_malformed(fixture: &LlamaFixture<'_>) { + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GRAMMAR_RETURNS_NULL, + ); + + assert_eq!( + LlamaSampler::grammar(fixture.model, "root ::= \"a\"", "root").err(), + Some(GrammarError::GrammarMalformed) + ); +} diff --git a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs index e2b654ae2..ca5bf01ee 100644 --- a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs +++ b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs @@ -3,17 +3,22 @@ use std::ffi::CStr; use anyhow::Context as _; use anyhow::Result; use anyhow::anyhow; +use llama_cpp_bindings::GrammarError; use llama_cpp_bindings::SampleError; use llama_cpp_bindings::SamplerAcceptError; +use llama_cpp_bindings::TokenToStringError; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::error::GrammarRuntimeError; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::llguidance_sampler::create_llg_sampler; use llama_cpp_bindings::model::AddBos; +use llama_cpp_bindings::model::LlamaModel; use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::params::LlamaModelParams; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_fault_injection::ArmedFault; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -648,3 +653,58 @@ fn sampling_through_a_grammar_in_an_error_state_reports_the_grammar_failure( Ok(()) } + +fn freshly_loaded(fixture: &LlamaFixture<'_>) -> Result { + Ok(LlamaModel::load_from_file( + fixture.backend, + fixture.model_path, + &LlamaModelParams::default(), + )?) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn creating_a_sampler_reports_a_vocabulary_llama_cpp_cannot_render( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = freshly_loaded(fixture)?; + let _fault = ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_TOKEN_TO_PIECE_FAILS); + + assert_eq!( + create_llg_sampler(&model, "regex", REGEX_GRAMMAR).err(), + Some(GrammarError::TokEnvUnavailable( + TokenToStringError::InsufficientBufferSpace(-34) + )) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn the_tokenizer_environment_reports_a_control_token_llama_cpp_cannot_render( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = freshly_loaded(fixture)?; + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SPECIAL_TOKEN_TO_PIECE_FAILS); + + assert_eq!( + model.approximate_tok_env().err(), + Some(TokenToStringError::InsufficientBufferSpace(-34)) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/logits_retrieval.rs b/llama-cpp-bindings-tests/tests/logits_retrieval.rs index 615a10a09..ba70f7bd7 100644 --- a/llama-cpp-bindings-tests/tests/logits_retrieval.rs +++ b/llama-cpp-bindings-tests/tests/logits_retrieval.rs @@ -5,6 +5,7 @@ use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::AddBos; use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_fault_injection::ArmedFault; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -89,3 +90,32 @@ fn get_logits_ith_returns_token_not_initialized_for_unknown_index( Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn reading_a_logits_row_reports_a_row_llama_cpp_does_not_return( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_GET_LOGITS_ITH_RETURNS_NULL); + + assert_eq!( + context.get_logits_ith(-1).err(), + Some(LogitsError::NullLogits) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/lora_adapters.rs b/llama-cpp-bindings-tests/tests/lora_adapters.rs index 6469728eb..d180b1d8b 100644 --- a/llama-cpp-bindings-tests/tests/lora_adapters.rs +++ b/llama-cpp-bindings-tests/tests/lora_adapters.rs @@ -1,6 +1,8 @@ use anyhow::Result; use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::error::LlamaLoraAdaptersError; use llama_cpp_bindings::model::lora_adapter_scale::LoraAdapterScale; +use llama_cpp_fault_injection::ArmedFault; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::download_model::download_model; use llama_cpp_test_harness::llama_test; @@ -61,3 +63,37 @@ fn set_lora_adapters_accepts_an_empty_adapter_set(fixture: &LlamaFixture<'_>) -> Ok(()) } + +#[llama_test( + model_source = HuggingFace( + "Qwen/Qwen2.5-7B-Instruct-GGUF", + "qwen2.5-7b-instruct-q2_k.gguf" + ), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn setting_lora_adapters_reports_a_status_llama_cpp_returns( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let adapter_path = download_model( + "ggml-org/LoRA-Deepthink-Reasoning-Qwen2.5-7B-Instruct-Q8_0-GGUF", + "Deepthink-Reasoning-Adapter-q8_0.gguf", + )?; + let adapter = fixture.model.lora_adapter_init(&adapter_path)?; + let context = fixture.build_context()?; + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SET_ADAPTERS_LORA_FAILS); + + assert_eq!( + context.set_lora_adapters(&[LoraAdapterScale { + adapter: &adapter, + scale: 1.0, + }]), + Err(LlamaLoraAdaptersError::ErrorResult(-1)) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/main.rs b/llama-cpp-bindings-tests/tests/main.rs index 39439a817..f69e45a48 100644 --- a/llama-cpp-bindings-tests/tests/main.rs +++ b/llama-cpp-bindings-tests/tests/main.rs @@ -4,6 +4,7 @@ mod batch_context_limits; mod chat_answer_parsing; mod chat_prompt_rendering; mod chat_session_input_validation; +mod chat_session_llama_cpp_failures; mod chat_template_lookup; mod chat_template_tool_call_support; mod completion_usage_accounting; diff --git a/llama-cpp-bindings-tests/tests/multimodal_audio.rs b/llama-cpp-bindings-tests/tests/multimodal_audio.rs index 27b903c16..27aadca42 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_audio.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_audio.rs @@ -125,7 +125,7 @@ fn transcribe_audio(fixture: &LlamaFixture<'_>, audio_file_name: &str) -> Result initial_position: n_past, max_generated_tokens: 512, } - .run()?; + .run(); Ok(outcome.generated.text().to_lowercase()) } diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs index a26972754..c52488745 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs @@ -3,6 +3,7 @@ use anyhow::Result; use anyhow::anyhow; use anyhow::bail; use llama_cpp_bindings::EvalMultimodalChunksParams; +use llama_cpp_bindings::FfiContractError; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::error::EvalMultimodalChunksError; use llama_cpp_bindings::error::sequence_id_out_of_range::SequenceIdOutOfRange; @@ -12,6 +13,7 @@ use llama_cpp_bindings::mtmd::MtmdBitmap; use llama_cpp_bindings::mtmd::MtmdContext; use llama_cpp_bindings::mtmd::MtmdEvalError; use llama_cpp_bindings::mtmd::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::MtmdInputChunkTypeError; use llama_cpp_bindings::mtmd::MtmdInputChunks; use llama_cpp_bindings::mtmd::MtmdInputText; use llama_cpp_bindings::mtmd::NonCausalChunkMicroBatchMismatch; @@ -21,6 +23,7 @@ use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; use llama_cpp_bindings_tests::media_question_chat_session::media_question_chat_session; use llama_cpp_bindings_tests::prime_kv_cache_with::prime_kv_cache_with; use llama_cpp_bindings_tests::prompt_question::PROMPT_QUESTION; +use llama_cpp_fault_injection::ArmedFault; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -564,7 +567,7 @@ fn gemma3_eval_chunks_rejects_a_non_causal_image_larger_than_the_micro_batch_bef .mtmd_context .expect("mmproj_file declared in attribute"); let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; - let image_tokens = ChunkTokenBreakdown::from_chunks(&chunks)?.image; + let image_tokens = ChunkTokenBreakdown::from_chunks(&chunks).image; let mut llama_ctx = LlamaContext::from_model( fixture.model, fixture.backend, @@ -694,7 +697,7 @@ fn classifier_reports_an_image_the_context_has_no_room_left_for( .mtmd_context .expect("mmproj_file declared in attribute"); let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; - let breakdown = ChunkTokenBreakdown::from_chunks(&chunks)?; + let breakdown = ChunkTokenBreakdown::from_chunks(&chunks); let mut llama_ctx = LlamaContext::from_model( fixture.model, fixture.backend, @@ -730,3 +733,132 @@ fn classifier_reports_an_image_the_context_has_no_room_left_for( Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 256, + n_ubatch = 256, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_chunks_reports_an_image_the_context_has_no_room_left_for( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; + let mut llama_ctx = fixture.build_context()?; + + prime_kv_cache_with(fixture, &mut llama_ctx, &"hello ".repeat(230))?; + + let primed_positions = llama_ctx.kv_cache_seq_pos_max(0)? + 1; + + assert_eq!( + chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, primed_positions, 0, 256, true), + Err(MtmdEvalError::EvalFailed { code: 1 }) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 256, + n_ubatch = 256, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn fitting_reports_a_chunk_llama_cpp_does_not_return(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNKS_GET_RETURNS_NULL, + ); + + assert_eq!( + chunks.fit_to_micro_batch(mtmd_ctx, 256), + Err(MtmdEvalError::FfiContract(FfiContractError { + operation: "mtmd_input_chunks_get", + detail: "returned a null chunk within the chunk count", + })) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 256, + n_ubatch = 256, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn fitting_reports_a_chunk_type_the_bindings_do_not_recognize( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNK_TYPE_UNKNOWN); + + assert_eq!( + chunks.fit_to_micro_batch(mtmd_ctx, 256), + Err(MtmdEvalError::UnknownChunkType(MtmdInputChunkTypeError( + llama_cpp_bindings_sys::MTMD_INPUT_CHUNK_TYPE_COUNT + ))) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 256, + n_ubatch = 256, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn classifier_reports_a_fitted_chunk_llama_cpp_no_longer_returns( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; + let mut llama_ctx = fixture.build_context()?; + let mut classifier = SampledTokenClassifier::undeterminable(fixture.model); + let _fault = ArmedFault::after_passing( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNKS_GET_RETURNS_NULL, + chunks.len(), + ); + + assert_eq!( + classifier.eval_multimodal_chunks( + &chunks, + mtmd_ctx, + &mut llama_ctx, + EvalMultimodalChunksParams { + start_position: 0, + seq_id: 0, + n_batch: 256, + logits_last: true, + }, + ), + Err(EvalMultimodalChunksError::ChunkOutOfBounds(0)) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs index b5e3518ef..4768eebc2 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs @@ -2,8 +2,12 @@ use anyhow::Result; use anyhow::anyhow; use llama_cpp_bindings::mtmd::MtmdBitmap; use llama_cpp_bindings::mtmd::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::MtmdInputChunksError; use llama_cpp_bindings::mtmd::MtmdInputText; +use llama_cpp_bindings::mtmd::MtmdTokenizeError; use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_fault_injection::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -315,3 +319,38 @@ fn tokenize_with_null_byte_in_text_returns_error(fixture: &LlamaFixture<'_>) { let result = mtmd_ctx.tokenize(input_text, &[]); assert!(result.is_err()); } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn tokenizing_reports_a_chunk_list_llama_cpp_cannot_allocate(fixture: &LlamaFixture<'_>) { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNKS_INIT_THROWS); + + assert_eq!( + mtmd_ctx + .tokenize( + MtmdInputText { + text: "Hello".to_owned(), + add_special: true, + parse_special: true, + }, + &[], + ) + .err(), + Some(MtmdTokenizeError::InputChunksError( + MtmdInputChunksError::Reported { + message: injected_fault_message(), + } + )) + ); +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs index d6591edac..b459dd012 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs @@ -100,7 +100,7 @@ fn image_and_audio_together(fixture: &LlamaFixture<'_>) -> Result<()> { .tokenize(input_text, &[&image_bitmap, &audio_bitmap]) .with_context(|| "failed to tokenize combined image and audio input")?; - let expected = ChunkTokenBreakdown::from_chunks(&chunks)?; + let expected = ChunkTokenBreakdown::from_chunks(&chunks); assert!( expected.image > 0, "image input must produce at least one image-chunk token" @@ -159,7 +159,7 @@ fn image_and_audio_together(fixture: &LlamaFixture<'_>) -> Result<()> { initial_position: n_past, max_generated_tokens: MAX_GENERATED_TOKENS, } - .run()?; + .run(); let description = outcome.generated.text().to_lowercase(); assert!( diff --git a/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs b/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs index 78665cce4..08e3ea8f6 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs @@ -7,11 +7,14 @@ use llama_cpp_bindings::model::ParseSpecialTokens; use llama_cpp_bindings::mtmd::MtmdBitmap; use llama_cpp_bindings::mtmd::MtmdInputChunkType; use llama_cpp_bindings::mtmd::MtmdInputChunks; +use llama_cpp_bindings::mtmd::MtmdInputChunksError; use llama_cpp_bindings::mtmd::MtmdInputPart; use llama_cpp_bindings::mtmd::MtmdInputTextPart; use llama_cpp_bindings::mtmd::MtmdTokenizeError; use llama_cpp_bindings::mtmd::mtmd_default_marker; use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_fault_injection::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -176,3 +179,58 @@ fn a_text_part_containing_a_nul_byte_is_rejected(fixture: &LlamaFixture<'_>) { .map(MtmdTokenizeError::CStringError) ); } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn tokenizing_parts_reports_a_chunk_list_llama_cpp_cannot_allocate(fixture: &LlamaFixture<'_>) { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNKS_INIT_THROWS); + + assert_eq!( + mtmd_ctx + .tokenize_from_parts(&[text_part("Hello", true)], true) + .err(), + Some(MtmdTokenizeError::InputChunksError( + MtmdInputChunksError::Reported { + message: injected_fault_message(), + } + )) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn tokenizing_parts_reports_a_tokenizer_llama_cpp_fails(fixture: &LlamaFixture<'_>) { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_MTMD_TOKENIZE_FROM_PARTS_THROWS, + ); + + assert_eq!( + mtmd_ctx + .tokenize_from_parts(&[text_part("Hello", true)], true) + .err(), + Some(MtmdTokenizeError::Reported { + message: injected_fault_message(), + }) + ); +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs index b420d9451..0c6a2f329 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs @@ -96,7 +96,7 @@ fn gemma4_classifier_emits_reasoning_for_multimodal_thinking_prompt( initial_position: n_past, max_generated_tokens: MAX_GENERATED_TOKENS, } - .run()?; + .run(); let usage = classifier.usage(); @@ -206,7 +206,7 @@ fn mistral3_classifier_emits_reasoning_for_multimodal_thinking_prompt( initial_position: n_past, max_generated_tokens: MAX_GENERATED_TOKENS, } - .run()?; + .run(); let usage = classifier.usage(); @@ -307,7 +307,7 @@ fn qwen35_classifier_emits_reasoning_for_multimodal_thinking_prompt( initial_position: n_past, max_generated_tokens: MAX_GENERATED_TOKENS, } - .run()?; + .run(); let usage = classifier.usage(); @@ -399,7 +399,7 @@ fn qwen36_classifier_emits_reasoning_for_multimodal_thinking_prompt( initial_position: n_past, max_generated_tokens: MAX_GENERATED_TOKENS, } - .run()?; + .run(); let usage = classifier.usage(); diff --git a/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs b/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs index e49d364b1..52f951562 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs @@ -41,7 +41,7 @@ fn build_multimodal_chunks_and_eval_into_usage( }; let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - let expected = ChunkTokenBreakdown::from_chunks(&chunks)?; + let expected = ChunkTokenBreakdown::from_chunks(&chunks); let context_params = (*fixture.context_params).into_llama_context_params(); let mut context = LlamaContext::from_model(model, fixture.backend, context_params)?; diff --git a/llama-cpp-bindings-tests/tests/multimodal_vision_inference.rs b/llama-cpp-bindings-tests/tests/multimodal_vision_inference.rs index adc23bdcf..2f90bf58c 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_vision_inference.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_vision_inference.rs @@ -69,7 +69,7 @@ fn multimodal_vision_inference_produces_output(fixture: &LlamaFixture<'_>) -> Re "tokenization should produce at least one chunk" ); - let expected = ChunkTokenBreakdown::from_chunks(&chunks)?; + let expected = ChunkTokenBreakdown::from_chunks(&chunks); eprintln!( "tokenized into {} chunks, text {} image {} audio {}", @@ -119,7 +119,7 @@ fn multimodal_vision_inference_produces_output(fixture: &LlamaFixture<'_>) -> Re initial_position: n_past, max_generated_tokens: 512, } - .run()?; + .run(); eprintln!("generated text: {}", totals.generated.text()); diff --git a/llama-cpp-bindings-tests/tests/reasoning_classification.rs b/llama-cpp-bindings-tests/tests/reasoning_classification.rs index c4f516ee3..63f8b01cc 100644 --- a/llama-cpp-bindings-tests/tests/reasoning_classification.rs +++ b/llama-cpp-bindings-tests/tests/reasoning_classification.rs @@ -27,17 +27,17 @@ const MAX_REASONING_ANSWER_TOKENS: i32 = 512; )] fn deepseek_r1_8b_answers_without_reasoning_after_its_reasoning_is_closed( fixture: &LlamaFixture<'_>, -) -> Result<()> { - let session = closed_reasoning_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]")?; +) { + let session = closed_reasoning_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]"); let answer = ChatAnswerGeneration { fixture, session: &session, - sampler: &mut penalized_nucleus_sampler(fixture.model)?, + sampler: &mut penalized_nucleus_sampler(fixture.model), max_generated_tokens: MAX_ANSWER_TOKENS, } - .run()?; + .run(); - answer.assert_content_without_reasoning(&session) + answer.assert_content_without_reasoning(&session); } #[llama_test( @@ -55,12 +55,14 @@ fn deepseek_r1_8b_reasoning_streams_match_the_parsed_answer_when_thinking_is_ena let answer = ChatAnswerGeneration { fixture, session: &session, - sampler: &mut penalized_nucleus_sampler(fixture.model)?, + sampler: &mut penalized_nucleus_sampler(fixture.model), max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, } - .run()?; + .run(); + + answer.assert_reasoning_matches_parse(&session); - answer.assert_reasoning_matches_parse(&session) + Ok(()) } #[llama_test( @@ -81,9 +83,11 @@ fn gemma4_answers_without_reasoning_when_thinking_is_disabled( sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_ANSWER_TOKENS, } - .run()?; + .run(); + + answer.assert_content_without_reasoning(&session); - answer.assert_content_without_reasoning(&session) + Ok(()) } #[llama_test( @@ -119,9 +123,11 @@ fn gemma4_reasoning_streams_match_the_parsed_answer_when_reasoning_is_prefilled( sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, } - .run()?; + .run(); - answer.assert_reasoning_matches_parse(&session) + answer.assert_reasoning_matches_parse(&session); + + Ok(()) } #[llama_test( @@ -139,12 +145,14 @@ fn glm47_answers_without_reasoning_when_thinking_is_disabled( let answer = ChatAnswerGeneration { fixture, session: &session, - sampler: &mut penalized_nucleus_sampler(fixture.model)?, + sampler: &mut penalized_nucleus_sampler(fixture.model), max_generated_tokens: MAX_ANSWER_TOKENS, } - .run()?; + .run(); + + answer.assert_content_without_reasoning(&session); - answer.assert_content_without_reasoning(&session) + Ok(()) } #[llama_test( @@ -162,12 +170,14 @@ fn glm47_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( let answer = ChatAnswerGeneration { fixture, session: &session, - sampler: &mut penalized_nucleus_sampler(fixture.model)?, + sampler: &mut penalized_nucleus_sampler(fixture.model), max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, } - .run()?; + .run(); + + answer.assert_reasoning_matches_parse(&session); - answer.assert_reasoning_matches_parse(&session) + Ok(()) } #[llama_test( @@ -181,16 +191,18 @@ fn glm47_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( fn mistral3_answers_without_reasoning_after_its_reasoning_is_closed( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let session = closed_reasoning_chat_session(fixture.model, SINGLE_WORD_INSTRUCTION, "[]")?; + let session = closed_reasoning_chat_session(fixture.model, SINGLE_WORD_INSTRUCTION, "[]"); let answer = ChatAnswerGeneration { fixture, session: &session, sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_ANSWER_TOKENS, } - .run()?; + .run(); - answer.assert_content_without_reasoning(&session) + answer.assert_content_without_reasoning(&session); + + Ok(()) } #[llama_test( @@ -211,9 +223,11 @@ fn mistral3_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, } - .run()?; + .run(); + + answer.assert_reasoning_matches_parse(&session); - answer.assert_reasoning_matches_parse(&session) + Ok(()) } #[llama_test( @@ -231,12 +245,14 @@ fn qwen35_answers_without_reasoning_when_thinking_is_disabled( let answer = ChatAnswerGeneration { fixture, session: &session, - sampler: &mut penalized_nucleus_sampler(fixture.model)?, + sampler: &mut penalized_nucleus_sampler(fixture.model), max_generated_tokens: MAX_ANSWER_TOKENS, } - .run()?; + .run(); + + answer.assert_content_without_reasoning(&session); - answer.assert_content_without_reasoning(&session) + Ok(()) } #[llama_test( @@ -254,12 +270,14 @@ fn qwen35_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( let answer = ChatAnswerGeneration { fixture, session: &session, - sampler: &mut penalized_nucleus_sampler(fixture.model)?, + sampler: &mut penalized_nucleus_sampler(fixture.model), max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, } - .run()?; + .run(); - answer.assert_reasoning_matches_parse(&session) + answer.assert_reasoning_matches_parse(&session); + + Ok(()) } #[llama_test( @@ -277,12 +295,14 @@ fn qwen36_answers_without_reasoning_when_thinking_is_disabled( let answer = ChatAnswerGeneration { fixture, session: &session, - sampler: &mut penalized_nucleus_sampler(fixture.model)?, + sampler: &mut penalized_nucleus_sampler(fixture.model), max_generated_tokens: MAX_ANSWER_TOKENS, } - .run()?; + .run(); + + answer.assert_content_without_reasoning(&session); - answer.assert_content_without_reasoning(&session) + Ok(()) } #[llama_test( @@ -300,10 +320,12 @@ fn qwen36_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( let answer = ChatAnswerGeneration { fixture, session: &session, - sampler: &mut penalized_nucleus_sampler(fixture.model)?, + sampler: &mut penalized_nucleus_sampler(fixture.model), max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, } - .run()?; + .run(); + + answer.assert_reasoning_matches_parse(&session); - answer.assert_reasoning_matches_parse(&session) + Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs b/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs index 0e30ee091..3c68dbdb3 100644 --- a/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs +++ b/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs @@ -4,17 +4,24 @@ use anyhow::bail; use llama_cpp_bindings::BareJsonToolCalls; use llama_cpp_bindings::GenerationProgress; use llama_cpp_bindings::IngestError; +use llama_cpp_bindings::ParseChatMessageError; +use llama_cpp_bindings::SampleError; use llama_cpp_bindings::SampledToken; use llama_cpp_bindings::SampledTokenClassifier; use llama_cpp_bindings::TokenOutsideVocabulary; use llama_cpp_bindings::TokenPiece; use llama_cpp_bindings::TokenToStringError; +use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::ingest_outcome::IngestOutcome; +use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::AddBos; use llama_cpp_bindings::model::ParseSpecialTokens; use llama_cpp_bindings::sampled_token_section::SampledTokenSection; +use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings::token::LlamaToken; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_fault_injection::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -243,3 +250,102 @@ fn ingesting_a_token_outside_the_vocabulary_reports_it_in_every_classifier( Ok(()) } + +fn failed_classification() -> IngestError { + IngestError::Classify(ParseChatMessageError::Reported { + message: injected_fault_message(), + }) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn ingesting_a_generated_token_reports_a_classification_llama_cpp_fails( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, "Hello!", "[]", false)?; + let mut classifier = + SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let generated = fixture + .model + .str_to_token("Hi", AddBos::Never, ParseSpecialTokens::Never)?; + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_CLASSIFY_THROWS); + + assert_eq!( + classifier.ingest(generated[0], &mut Vec::new()), + Err(failed_classification()) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn ending_the_generation_reports_a_classification_llama_cpp_fails( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, "Hello!", "[]", false)?; + let mut classifier = + SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let end_of_sequence = fixture + .model + .token_eos() + .context("the model declares an EOS token")?; + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_CLASSIFY_THROWS); + + assert_eq!( + classifier.ingest(end_of_sequence, &mut Vec::new()), + Err(failed_classification()) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn sampling_reports_a_classification_llama_cpp_fails(fixture: &LlamaFixture<'_>) -> Result<()> { + let session = user_turn_chat_session(fixture.model, "Hello!", "[]", false)?; + let mut classifier = + SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let mut context = fixture.build_context()?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + + classifier.feed_prompt_sequence_to_batch(&mut batch, &session.tokenize_prompt()?, 0, false)?; + context.decode(&batch)?; + + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_CLASSIFY_THROWS); + + assert_eq!( + classifier + .sample( + &mut LlamaSampler::greedy()?, + &context, + batch.n_tokens() - 1, + &mut Vec::new(), + ) + .err(), + Some(SampleError::Ingest(failed_classification())) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/tool_call_classification.rs b/llama-cpp-bindings-tests/tests/tool_call_classification.rs index 09561b892..678f46c73 100644 --- a/llama-cpp-bindings-tests/tests/tool_call_classification.rs +++ b/llama-cpp-bindings-tests/tests/tool_call_classification.rs @@ -48,8 +48,8 @@ fn qwen35_reasoning_closed_by_a_tool_call_is_attributed_to_each_section( let answer = format!("{REASONING_BEFORE_THE_CALL}{QWEN_XML_PARIS_WEATHER_CALL}"); let mut classifier = SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); - let outcomes = ingested_generation(&mut classifier, fixture.model, &answer)?; - let parsed = recognized_answer(&session, &answer, ChatOutputCompleteness::Finished)?; + let outcomes = ingested_generation(&mut classifier, fixture.model, &answer); + let parsed = recognized_answer(&session, &answer, ChatOutputCompleteness::Finished); assert_eq!( raw_text_where(&outcomes, |token| matches!( diff --git a/llama-cpp-bindings-tests/tests/tool_call_grammar.rs b/llama-cpp-bindings-tests/tests/tool_call_grammar.rs index fb1b389b1..646762d61 100644 --- a/llama-cpp-bindings-tests/tests/tool_call_grammar.rs +++ b/llama-cpp-bindings-tests/tests/tool_call_grammar.rs @@ -1,43 +1,19 @@ use anyhow::Context as _; use anyhow::Result; -use llama_cpp_bindings::ChatAssistantTurn; use llama_cpp_bindings::ChatOutputCompleteness; use llama_cpp_bindings::ChatSession; -use llama_cpp_bindings::ChatSessionOptions; use llama_cpp_bindings::ChatToolChoice; -use llama_cpp_bindings::model::LlamaChatMessage; -use llama_cpp_bindings::model::LlamaModel; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::chat_answer_generation::ChatAnswerGeneration; -use llama_cpp_bindings_tests::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; use llama_cpp_bindings_tests::paris_weather_tool_call::paris_weather_tool_call; use llama_cpp_bindings_tests::recognized_answer::recognized_answer; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_bindings_tests::weather_chat_session::weather_chat_session; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; const MAX_TOOL_CALL_TOKENS: i32 = 128; -fn weather_session_with_tool_choice( - model: &LlamaModel, - tool_choice: ChatToolChoice, -) -> Result> { - Ok(ChatSession::new( - model, - &[LlamaChatMessage::new( - "user".to_owned(), - "What is the weather in Paris?".to_owned(), - )?], - GET_WEATHER_TOOLS_JSON, - &ChatSessionOptions { - assistant_turn: ChatAssistantTurn::New, - enable_thinking: false, - parallel_tool_calls: false, - tool_choice, - }, - )?) -} - fn generated_tool_call_answer( fixture: &LlamaFixture<'_>, session: &ChatSession<'_>, @@ -52,7 +28,7 @@ fn generated_tool_call_answer( sampler: &mut sampler, max_generated_tokens: MAX_TOOL_CALL_TOKENS, } - .run()?; + .run(); Ok(answer.outcome.generated.text().to_owned()) } @@ -82,7 +58,7 @@ fn a_chat_session_without_tools_needs_no_grammar_sampler(fixture: &LlamaFixture< n_ubatch = 128, )] fn a_tool_choice_of_none_needs_no_grammar_sampler(fixture: &LlamaFixture<'_>) -> Result<()> { - let session = weather_session_with_tool_choice(fixture.model, ChatToolChoice::None)?; + let session = weather_chat_session(fixture.model, ChatToolChoice::None); assert!(session.grammar_sampler()?.is_none()); @@ -100,12 +76,12 @@ fn a_tool_choice_of_none_needs_no_grammar_sampler(fixture: &LlamaFixture<'_>) -> fn qwen36_required_tool_call_grammar_constrains_the_answer_to_a_tool_call( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let session = weather_session_with_tool_choice(fixture.model, ChatToolChoice::Required)?; + let session = weather_chat_session(fixture.model, ChatToolChoice::Required); let answer = generated_tool_call_answer(fixture, &session)?; assert!(!session.grammar_is_lazy()); assert_eq!( - recognized_answer(&session, &answer, ChatOutputCompleteness::Finished)?.tool_calls, + recognized_answer(&session, &answer, ChatOutputCompleteness::Finished).tool_calls, [paris_weather_tool_call()] ); @@ -123,12 +99,12 @@ fn qwen36_required_tool_call_grammar_constrains_the_answer_to_a_tool_call( fn qwen36_automatic_tool_call_grammar_constrains_the_tool_call_after_its_trigger( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let session = weather_session_with_tool_choice(fixture.model, ChatToolChoice::Auto)?; + let session = weather_chat_session(fixture.model, ChatToolChoice::Auto); let answer = generated_tool_call_answer(fixture, &session)?; assert!(session.grammar_is_lazy()); assert_eq!( - recognized_answer(&session, &answer, ChatOutputCompleteness::Finished)?.tool_calls, + recognized_answer(&session, &answer, ChatOutputCompleteness::Finished).tool_calls, [paris_weather_tool_call()] ); diff --git a/llama-cpp-bindings-tests/tests/tool_call_parsing.rs b/llama-cpp-bindings-tests/tests/tool_call_parsing.rs index 724ac869b..b50502c58 100644 --- a/llama-cpp-bindings-tests/tests/tool_call_parsing.rs +++ b/llama-cpp-bindings-tests/tests/tool_call_parsing.rs @@ -66,7 +66,7 @@ fn assert_parses_to_paris_weather_call(fixture: &LlamaFixture<'_>, answer: &str) GET_WEATHER_TOOLS_JSON, false, )?; - let parsed = recognized_answer(&session, answer, ChatOutputCompleteness::Finished)?; + let parsed = recognized_answer(&session, answer, ChatOutputCompleteness::Finished); assert_eq!(parsed.tool_calls, [paris_weather_tool_call()]); @@ -163,7 +163,7 @@ tuna\n\ \n\ ", ChatOutputCompleteness::Finished, - )?; + ); assert_eq!( parsed.tool_calls, @@ -202,7 +202,7 @@ fn qwen35_parses_a_tool_call_cut_inside_its_arguments_as_a_named_call( &session, "\n\n) -> QWEN_XML_PARIS_WEATHER_CALL.replace("Paris", "Berlin") ), ChatOutputCompleteness::Finished, - )?; + ); assert_eq!( parsed.tool_calls, @@ -292,7 +292,7 @@ fn plain_content_answering_a_tool_request_parses_without_tool_calls( &session, PLAIN_CONTENT_REFUSAL, ChatOutputCompleteness::Finished, - )?; + ); assert_eq!(parsed.tool_calls, []); @@ -311,7 +311,7 @@ fn deepseek_r1_8b_plain_content_without_tools_parses_without_tool_calls( fixture: &LlamaFixture<'_>, ) -> Result<()> { let session = user_turn_chat_session(fixture.model, "Hello!", "[]", false)?; - let parsed = recognized_answer(&session, "Hello there.", ChatOutputCompleteness::Finished)?; + let parsed = recognized_answer(&session, "Hello there.", ChatOutputCompleteness::Finished); assert_eq!(parsed.tool_calls, []); diff --git a/llama-cpp-bindings/Cargo.toml b/llama-cpp-bindings/Cargo.toml index 91d4eeb11..c73ccb3eb 100644 --- a/llama-cpp-bindings/Cargo.toml +++ b/llama-cpp-bindings/Cargo.toml @@ -26,6 +26,7 @@ toktrie = { workspace = true } [dev-dependencies] anyhow = { workspace = true } +llama-cpp-fault-injection = { workspace = true } llama-cpp-wrapper-error-fixture = { workspace = true } serial_test = { workspace = true } diff --git a/llama-cpp-bindings/src/chat_message_parse_outcome.rs b/llama-cpp-bindings/src/chat_message_parse_outcome.rs index 702d0f0a8..70f0f479c 100644 --- a/llama-cpp-bindings/src/chat_message_parse_outcome.rs +++ b/llama-cpp-bindings/src/chat_message_parse_outcome.rs @@ -10,6 +10,18 @@ pub enum ChatMessageParseOutcome { } impl ChatMessageParseOutcome { + /// The recognized message, or the raw answer the parser did not recognize. + /// + /// # Errors + /// + /// Returns the [`RawChatMessage`] when the answer does not match the template's format. + pub fn into_recognized(self) -> Result { + match self { + Self::Recognized(message) => Ok(message), + Self::Unrecognized(raw) => Err(raw), + } + } + /// The message this answer becomes when `tool_calls` were recognized after it: a recognized /// answer keeps its content and reasoning, an unrecognized one is kept whole as content. #[must_use] @@ -76,4 +88,30 @@ mod tests { ) ); } + + #[test] + fn a_recognized_answer_is_its_message() { + let message = ParsedChatMessage::new("Paris.".to_owned(), String::new(), Vec::new()); + + assert_eq!( + ChatMessageParseOutcome::Recognized(message.clone()).into_recognized(), + Ok(message) + ); + } + + fn odd_answer() -> RawChatMessage { + RawChatMessage { + text: "Paris.".to_owned(), + completeness: ChatOutputCompleteness::Finished, + ffi_error_message: "the format differs".to_owned(), + } + } + + #[test] + fn an_unrecognized_answer_is_its_raw_text() { + assert_eq!( + ChatMessageParseOutcome::Unrecognized(odd_answer()).into_recognized(), + Err(odd_answer()) + ); + } } diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index e9f9ca80b..ac27e7b9c 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -983,7 +983,7 @@ impl<'model> LlamaContext<'model> { self.produced_logits()?; let row = NonNull::new(unsafe { - llama_cpp_bindings_sys::llama_get_logits_ith(self.context.as_ptr(), token_index) + llama_cpp_bindings_sys::llama_rs_get_logits_ith(self.context.as_ptr(), token_index) }) .ok_or(LogitsError::NullLogits)?; @@ -1020,7 +1020,7 @@ impl<'model> LlamaContext<'model> { .map_or(ptr::null_mut(), ptr::from_mut); let scales_ptr = scales.first_mut().map_or(ptr::null_mut(), ptr::from_mut); let err_code = unsafe { - llama_cpp_bindings_sys::llama_set_adapters_lora( + llama_cpp_bindings_sys::llama_rs_set_adapters_lora( self.context.as_ptr(), raw_adapters_ptr, raw_adapters.len(), diff --git a/llama-cpp-bindings/src/error/eval_multimodal_chunks_error.rs b/llama-cpp-bindings/src/error/eval_multimodal_chunks_error.rs index 7874616f1..560436ebc 100644 --- a/llama-cpp-bindings/src/error/eval_multimodal_chunks_error.rs +++ b/llama-cpp-bindings/src/error/eval_multimodal_chunks_error.rs @@ -1,6 +1,6 @@ use crate::mtmd::MtmdEvalError; -#[derive(Debug, thiserror::Error)] +#[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum EvalMultimodalChunksError { #[error("{0}")] EvalFailed(#[from] MtmdEvalError), diff --git a/llama-cpp-bindings/src/error/llama_cpp_error.rs b/llama-cpp-bindings/src/error/llama_cpp_error.rs index e40664a8a..12341da21 100644 --- a/llama-cpp-bindings/src/error/llama_cpp_error.rs +++ b/llama-cpp-bindings/src/error/llama_cpp_error.rs @@ -8,7 +8,7 @@ use crate::error::json_schema_to_grammar_error::JsonSchemaToGrammarError; use crate::error::llama_context_load_error::LlamaContextLoadError; use crate::error::llama_model_load_error::LlamaModelLoadError; -#[derive(Debug, thiserror::Error)] +#[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum LlamaCppError { #[error("BackendAlreadyInitialized")] BackendAlreadyInitialized, diff --git a/llama-cpp-bindings/src/lazy_grammar_triggers.rs b/llama-cpp-bindings/src/lazy_grammar_triggers.rs index 74a46ea51..7bdae2525 100644 --- a/llama-cpp-bindings/src/lazy_grammar_triggers.rs +++ b/llama-cpp-bindings/src/lazy_grammar_triggers.rs @@ -129,6 +129,9 @@ impl LazyGrammarTriggers { mod tests { use std::ptr; + use llama_cpp_fault_injection::ArmedFault; + use llama_cpp_fault_injection::injected_fault_message; + use super::LazyGrammarTriggers; use super::anchored_pattern; use super::regex_escape_status_to_result; @@ -255,4 +258,17 @@ mod tests { }) ); } + + #[test] + fn a_trigger_word_llama_cpp_cannot_escape_is_reported() { + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_REGEX_ESCAPE_THROWS); + + assert_eq!( + LazyGrammarTriggers::of(vec![ChatGrammarTrigger::Word("".to_owned())]), + Err(GrammarError::Reported { + message: injected_fault_message(), + }) + ); + } } diff --git a/llama-cpp-bindings/src/llguidance_sampler.rs b/llama-cpp-bindings/src/llguidance_sampler.rs index a55228044..43fd7fbb5 100644 --- a/llama-cpp-bindings/src/llguidance_sampler.rs +++ b/llama-cpp-bindings/src/llguidance_sampler.rs @@ -3,7 +3,9 @@ use std::sync::Arc; use llama_cpp_error_recorder::RecordedError; use llama_cpp_error_recorder::record; +use llguidance::earley::SlicedBiasComputer; use toktrie::ApproximateTokEnv; +use toktrie::InferenceCapabilities; use crate::GrammarError; use crate::grammar_matcher::GrammarMatcher; @@ -106,6 +108,22 @@ static mut LLG_SAMPLER_I: llama_cpp_bindings_sys::llama_sampler_i = copy_state: None, }; +const UNCOMPILABLE_BIAS_SLICE: &str = "("; + +fn bias_slices() -> Vec { + let factory_fault_reached = unsafe { + llama_cpp_bindings_sys::llama_rs_fault_point_reached( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_LLGUIDANCE_FACTORY_FAILS, + ) + }; + + if factory_fault_reached { + return vec![UNCOMPILABLE_BIAS_SLICE.to_owned()]; + } + + SlicedBiasComputer::general_slices() +} + fn create_llg_sampler_from_tok_env( tok_env: Arc, grammar_kind: &str, @@ -113,10 +131,13 @@ fn create_llg_sampler_from_tok_env( ) -> Result { let tok_env_dyn: Arc = tok_env.clone(); - let factory = llguidance::ParserFactory::new_simple(&tok_env_dyn).map_err(|factory_error| { - GrammarError::LlguidanceFactoryUnavailable { - message: factory_error.to_string(), - } + let factory = llguidance::ParserFactory::new( + &tok_env_dyn, + InferenceCapabilities::default(), + &bias_slices(), + ) + .map_err(|factory_error| GrammarError::LlguidanceFactoryUnavailable { + message: factory_error.to_string(), })?; let grammar = llguidance::api::TopLevelGrammar::from_tagged_str(grammar_kind, grammar_data) @@ -171,6 +192,10 @@ mod tests { use toktrie::TokRxInfo; use toktrie::TokTrie; + use llama_cpp_fault_injection::ArmedFault; + use toktrie::InferenceCapabilities; + + use super::UNCOMPILABLE_BIAS_SLICE; use super::create_llg_sampler_from_tok_env; use crate::GrammarError; use crate::SampleError; @@ -185,14 +210,17 @@ mod tests { const LETTER_B: LlamaToken = LlamaToken::new(1); const OUT_OF_VOCABULARY: LlamaToken = LlamaToken::new(2); - fn letter_a_sampler() -> Result { + fn letters() -> Arc { let vocabulary_without_end_of_generation = TokRxInfo::new(2, INVALID_TOKEN); - let letters = TokTrie::from( + + Arc::new(ApproximateTokEnv::new(TokTrie::from( &vocabulary_without_end_of_generation, &[b"a".to_vec(), b"b".to_vec()], - ); + ))) + } - create_llg_sampler_from_tok_env(Arc::new(ApproximateTokEnv::new(letters)), "regex", "a") + fn letter_a_sampler() -> Result { + create_llg_sampler_from_tok_env(letters(), "regex", "a") } fn letter_candidates() -> LlamaTokenDataArray { @@ -283,4 +311,23 @@ mod tests { }) ); } + + #[test] + fn creating_a_sampler_reports_a_parser_factory_llguidance_cannot_build() { + let letters_env: Arc = letters(); + let factory_failure = llguidance::ParserFactory::new( + &letters_env, + InferenceCapabilities::default(), + &[UNCOMPILABLE_BIAS_SLICE.to_owned()], + ) + .err() + .map(|factory_error| GrammarError::LlguidanceFactoryUnavailable { + message: factory_error.to_string(), + }); + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_LLGUIDANCE_FACTORY_FAILS); + + assert_ne!(factory_failure, None); + assert_eq!(letter_a_sampler().err(), factory_failure); + } } diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index ddcd76a60..1a1c4c915 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -65,8 +65,8 @@ fn validate_string_length_for_tokenizer(length: usize) -> Result Result { + let length = validate_string_length_for_tokenizer(text.len())?; let c_string = CString::new(text)?; - let length = validate_string_length_for_tokenizer(c_string.as_bytes().len())?; Ok(TokenizerInput { text: c_string, @@ -359,7 +359,7 @@ impl LlamaModel { let mut buffer: Vec = vec![0u8; buffer_size]; let lstrip = lstrip.map_or(0, |strip_count| i32::from(strip_count.get())); let size = unsafe { - llama_cpp_bindings_sys::llama_token_to_piece( + llama_cpp_bindings_sys::llama_rs_token_to_piece( self.vocab_ptr(), token.0, buffer.as_mut_ptr().cast::(), @@ -874,13 +874,32 @@ impl Drop for LlamaModel { #[cfg(test)] mod cstring_with_validated_len_tests { + use std::ffi::CString; + use std::str; + use super::cstring_with_validated_len; + use crate::StringToTokenError; #[test] fn cstring_with_validated_len_null_byte_returns_error() { - let result = cstring_with_validated_len("null\0byte"); + assert_eq!( + cstring_with_validated_len("null\0byte").err(), + CString::new("null\0byte") + .err() + .map(StringToTokenError::NulError) + ); + } + + #[test] + fn a_text_longer_than_a_c_int_is_refused_before_it_is_copied() { + let too_long = vec![0_u8; usize::try_from(i32::MAX).unwrap() + 1]; - assert!(result.is_err()); + assert_eq!( + cstring_with_validated_len(str::from_utf8(&too_long).unwrap()).err(), + i32::try_from(too_long.len()) + .err() + .map(StringToTokenError::CIntConversionError) + ); } } @@ -890,9 +909,10 @@ mod validate_string_length_for_tokenizer_tests { #[test] fn validate_string_length_overflow_returns_error() { - let result = validate_string_length_for_tokenizer(usize::MAX); - - assert!(result.is_err()); + assert_eq!( + validate_string_length_for_tokenizer(usize::MAX), + i32::try_from(usize::MAX).map_err(crate::StringToTokenError::CIntConversionError) + ); } } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs index 52387656a..3d90e885a 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs @@ -120,8 +120,9 @@ impl MtmdInputChunk { /// # Errors /// Returns an error if the chunk type is unknown. pub fn chunk_type(&self) -> Result { - let chunk_type = - unsafe { llama_cpp_bindings_sys::mtmd_input_chunk_get_type(self.chunk.as_ptr()) }; + let chunk_type = unsafe { + llama_cpp_bindings_sys::llama_rs_mtmd_input_chunk_get_type(self.chunk.as_ptr()) + }; MtmdInputChunkType::try_from(chunk_type) } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs index 74ef98245..a365f0d7e 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs @@ -111,8 +111,9 @@ impl MtmdInputChunks { return None; } - let chunk_ptr = - unsafe { llama_cpp_bindings_sys::mtmd_input_chunks_get(self.chunks.as_ptr(), index) }; + let chunk_ptr = unsafe { + llama_cpp_bindings_sys::llama_rs_mtmd_input_chunks_get(self.chunks.as_ptr(), index) + }; NonNull::new(chunk_ptr.cast_mut()).map(|ptr| MtmdInputChunk { chunk: ptr, diff --git a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs index 0fcb69dba..2d3120407 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs @@ -186,6 +186,9 @@ mod tests { use std::num::NonZeroU32; use std::ptr; + use llama_cpp_fault_injection::ArmedFault; + use llama_cpp_fault_injection::injected_fault_message; + use super::MtmdMemoryUsage; use super::get_memory_usage_status_to_result; use crate::mtmd::mtmd_context_params::MtmdContextParams; @@ -240,6 +243,21 @@ mod tests { ); } + #[test] + fn a_measurement_llama_cpp_fails_is_reported() { + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_MTMD_GET_MEMORY_USAGE_THROWS, + ); + + assert_eq!( + MtmdMemoryUsage::measure(c"/nonexistent/mmproj.gguf", &MtmdContextParams::default()) + .err(), + Some(MtmdMemoryUsageError::Reported { + message: injected_fault_message(), + }) + ); + } + #[test] fn measuring_with_unrepresentable_params_is_refused() { let params = MtmdContextParams { diff --git a/llama-cpp-bindings/src/sampling.rs b/llama-cpp-bindings/src/sampling.rs index 288b68c6b..550688c38 100644 --- a/llama-cpp-bindings/src/sampling.rs +++ b/llama-cpp-bindings/src/sampling.rs @@ -1221,12 +1221,32 @@ mod tests { #[cfg(test)] mod checked_usize_as_i32_sampling_tests { + + use super::LlamaSampler; use super::checked_usize_as_i32_sampling; + use crate::error::SamplingError; + use crate::token::logit_bias::LlamaLogitBias; #[test] fn checked_usize_as_i32_sampling_overflow() { - let result = checked_usize_as_i32_sampling(usize::MAX); - assert!(result.is_err()); + assert_eq!( + checked_usize_as_i32_sampling(usize::MAX), + i32::try_from(usize::MAX).map_err(SamplingError::IntegerOverflow) + ); + } + + #[test] + fn a_logit_bias_list_longer_than_a_c_int_is_refused() { + let bias_count = usize::try_from(i32::MAX).unwrap() + 1; + let zeroed_biases = + unsafe { Box::<[LlamaLogitBias]>::new_zeroed_slice(bias_count).assume_init() }; + + assert_eq!( + LlamaSampler::logit_bias(1, &zeroed_biases).err(), + i32::try_from(bias_count) + .err() + .map(SamplingError::IntegerOverflow) + ); } } diff --git a/llama-cpp-bindings/src/token/data_array.rs b/llama-cpp-bindings/src/token/data_array.rs index df5362e11..4d9b9fc16 100644 --- a/llama-cpp-bindings/src/token/data_array.rs +++ b/llama-cpp-bindings/src/token/data_array.rs @@ -220,8 +220,12 @@ impl LlamaTokenDataArray { #[cfg(test)] mod tests { + use llama_cpp_fault_injection::ArmedFault; + use llama_cpp_fault_injection::injected_fault_message; + use super::LlamaTokenDataArray; use crate::error::SamplerApplyError; + use crate::error::SamplingError; use crate::error::TokenSamplingError; use crate::token::LlamaToken; use crate::token::data::LlamaTokenData; @@ -340,6 +344,42 @@ mod tests { ); } + #[test] + fn sampling_reports_a_selecting_sampler_llama_cpp_cannot_build() { + let distribution_failure = { + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_INIT_DIST_THROWS, + ); + + excluded_candidates().sample_token(7) + }; + let greedy_failure = { + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GREEDY_THROWS, + ); + + excluded_candidates().sample_token_greedy() + }; + + assert_eq!( + [distribution_failure, greedy_failure], + [ + Err(TokenSamplingError::SamplerInitialization( + SamplingError::Reported { + sampler: "distribution", + message: injected_fault_message(), + } + )), + Err(TokenSamplingError::SamplerInitialization( + SamplingError::Reported { + sampler: "greedy", + message: injected_fault_message(), + } + )), + ] + ); + } + #[test] fn apply_greedy_sampler_selects_highest_logit() { use crate::sampling::LlamaSampler; diff --git a/llama-cpp-bindings/src/tool_call_format/json_object.rs b/llama-cpp-bindings/src/tool_call_format/json_object.rs index c82a322af..40be6d2ce 100644 --- a/llama-cpp-bindings/src/tool_call_format/json_object.rs +++ b/llama-cpp-bindings/src/tool_call_format/json_object.rs @@ -9,11 +9,7 @@ fn try_parse_one_object( input: &str, shape: &JsonObjectShape, ) -> Result, JsonObjectFailure> { - let trimmed_start = input.find('{'); - let Some(start) = trimmed_start else { - return Ok(None); - }; - + let start = input.find('{').unwrap_or(input.len()); let mut stream = serde_json::Deserializer::from_str(&input[start..]) .into_iter::>(); let map = match stream.next() { diff --git a/llama-cpp-fault-injection/Cargo.toml b/llama-cpp-fault-injection/Cargo.toml new file mode 100644 index 000000000..0e3afbbb2 --- /dev/null +++ b/llama-cpp-fault-injection/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "llama-cpp-fault-injection" +description = "Arms the C++ wrapper fault points so tests can drive the paths llama.cpp only takes when it fails" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +[dependencies] +llama-cpp-bindings-sys = { workspace = true, features = ["fault-injection"] } + +[lints] +workspace = true diff --git a/llama-cpp-fault-injection/src/armed_fault.rs b/llama-cpp-fault-injection/src/armed_fault.rs new file mode 100644 index 000000000..dce628b6a --- /dev/null +++ b/llama-cpp-fault-injection/src/armed_fault.rs @@ -0,0 +1,36 @@ +use std::marker::PhantomData; + +/// Makes one C++ wrapper fault point fail on the current thread until the guard is dropped. +/// +/// The fault point is armed per thread, so the guard cannot leave the thread that armed it. +#[derive(Debug)] +pub struct ArmedFault { + armed_on_this_thread: PhantomData<*const ()>, +} + +impl ArmedFault { + /// Fails every time the wrappers reach `point`. + #[must_use] + pub fn at(point: llama_cpp_bindings_sys::llama_rs_fault_point) -> Self { + Self::after_passing(point, 0) + } + + /// Lets the wrappers pass `point` `passing_hits` times, then fails every later time. + #[must_use] + pub fn after_passing( + point: llama_cpp_bindings_sys::llama_rs_fault_point, + passing_hits: usize, + ) -> Self { + unsafe { llama_cpp_bindings_sys::llama_rs_fault_injection_arm(point, passing_hits) }; + + Self { + armed_on_this_thread: PhantomData, + } + } +} + +impl Drop for ArmedFault { + fn drop(&mut self) { + unsafe { llama_cpp_bindings_sys::llama_rs_fault_injection_disarm() }; + } +} diff --git a/llama-cpp-fault-injection/src/injected_fault_message.rs b/llama-cpp-fault-injection/src/injected_fault_message.rs new file mode 100644 index 000000000..aefddd74e --- /dev/null +++ b/llama-cpp-fault-injection/src/injected_fault_message.rs @@ -0,0 +1,14 @@ +use std::ffi::CStr; + +/// The message of the C++ exception an armed fault point throws. +/// +/// # Panics +/// +/// Panics when the wrappers' fault message is not UTF-8. +#[must_use] +pub fn injected_fault_message() -> String { + unsafe { CStr::from_ptr(llama_cpp_bindings_sys::llama_rs_fault_injection_message()) } + .to_str() + .expect("the wrappers' fault message must be UTF-8") + .to_owned() +} diff --git a/llama-cpp-fault-injection/src/lib.rs b/llama-cpp-fault-injection/src/lib.rs new file mode 100644 index 000000000..9dc8ab6bb --- /dev/null +++ b/llama-cpp-fault-injection/src/lib.rs @@ -0,0 +1,5 @@ +pub mod armed_fault; +pub mod injected_fault_message; + +pub use armed_fault::ArmedFault; +pub use injected_fault_message::injected_fault_message; diff --git a/llama-cpp-test-harness/src/harness_run_error.rs b/llama-cpp-test-harness/src/harness_run_error.rs index 55a21c5bd..fabee8e78 100644 --- a/llama-cpp-test-harness/src/harness_run_error.rs +++ b/llama-cpp-test-harness/src/harness_run_error.rs @@ -3,7 +3,7 @@ use thiserror::Error; use crate::harness_arguments_error::HarnessArgumentsError; -#[derive(Debug, Error)] +#[derive(Debug, Eq, Error, PartialEq)] pub enum HarnessRunError { #[error("failed to parse harness arguments: {0}")] ArgumentParsing(#[from] HarnessArgumentsError), diff --git a/llama-cpp-test-harness/src/run.rs b/llama-cpp-test-harness/src/run.rs index 69ffb165d..569137630 100644 --- a/llama-cpp-test-harness/src/run.rs +++ b/llama-cpp-test-harness/src/run.rs @@ -28,6 +28,7 @@ mod tests { use std::process::ExitCode; use libtest_mimic::Conclusion; + use llama_cpp_bindings::error::LlamaCppError; use llama_cpp_bindings::llama_backend::LlamaBackend; use crate::harness_run_error::HarnessRunError; @@ -97,7 +98,12 @@ mod tests { let outcome = run_to_conclusions(); - assert!(matches!(outcome, Err(HarnessRunError::BackendInit(_)))); + assert_eq!( + outcome.err(), + Some(HarnessRunError::BackendInit( + LlamaCppError::BackendAlreadyInitialized + )) + ); } #[test] diff --git a/llama-cpp-wrapper-sources/src/compile_commands_file.rs b/llama-cpp-wrapper-sources/src/compile_commands_file.rs index 62bf71bd3..380aec0c2 100644 --- a/llama-cpp-wrapper-sources/src/compile_commands_file.rs +++ b/llama-cpp-wrapper-sources/src/compile_commands_file.rs @@ -4,6 +4,7 @@ use std::path::PathBuf; use crate::compile_command::CompileCommand; use crate::cpp_standard::CPP_STANDARD; +use crate::fault_injection_define::FAULT_INJECTION_DEFINE; use crate::vendored_include_dirs::VENDORED_INCLUDE_DIRS; use crate::wrapper_include_dirs::WRAPPER_INCLUDE_DIRS; use crate::wrapper_source_paths::wrapper_source_paths; @@ -26,7 +27,11 @@ impl CompileCommandsFile { wrapper_source_paths(&self.sys_dir) .into_iter() .map(|file| { - let mut arguments = vec![CPP_COMPILER.to_owned(), format!("-std={CPP_STANDARD}")]; + let mut arguments = vec![ + CPP_COMPILER.to_owned(), + format!("-std={CPP_STANDARD}"), + format!("-D{FAULT_INJECTION_DEFINE}"), + ]; for include_dir in WRAPPER_INCLUDE_DIRS { arguments.push(format!("-I{}", absolute_path(&self.sys_dir, include_dir))); @@ -159,7 +164,7 @@ mod tests { } #[test] - fn the_compiler_and_standard_lead_the_command_line() { + fn the_compiler_standard_and_fault_injection_lead_the_command_line() { let commands = database_for("/unused").commands(); assert_eq!( @@ -168,11 +173,15 @@ mod tests { .map(|command| command .arguments .iter() - .take(2) + .take(3) .cloned() .collect::>()) .unwrap_or_default(), - vec!["c++".to_owned(), "-std=c++17".to_owned()] + vec![ + "c++".to_owned(), + "-std=c++17".to_owned(), + "-DLLAMA_RS_FAULT_INJECTION".to_owned() + ] ); } diff --git a/llama-cpp-wrapper-sources/src/fault_injection_define.rs b/llama-cpp-wrapper-sources/src/fault_injection_define.rs new file mode 100644 index 000000000..56cb4db5c --- /dev/null +++ b/llama-cpp-wrapper-sources/src/fault_injection_define.rs @@ -0,0 +1 @@ +pub const FAULT_INJECTION_DEFINE: &str = "LLAMA_RS_FAULT_INJECTION"; diff --git a/llama-cpp-wrapper-sources/src/lib.rs b/llama-cpp-wrapper-sources/src/lib.rs index 512649a46..996af00cd 100644 --- a/llama-cpp-wrapper-sources/src/lib.rs +++ b/llama-cpp-wrapper-sources/src/lib.rs @@ -11,6 +11,7 @@ pub mod compile_command; pub mod compile_commands_file; pub mod cpp_standard; +pub mod fault_injection_define; pub mod vendored_include_dirs; pub mod wrapper_headers; pub mod wrapper_include_dirs; diff --git a/llama-cpp-wrapper-sources/src/wrapper_headers.rs b/llama-cpp-wrapper-sources/src/wrapper_headers.rs index 958275a2a..69010918a 100644 --- a/llama-cpp-wrapper-sources/src/wrapper_headers.rs +++ b/llama-cpp-wrapper-sources/src/wrapper_headers.rs @@ -6,6 +6,7 @@ pub const WRAPPER_HEADERS: &[&str] = &[ "wrapper_chat_templates_handle.h", "wrapper_common.h", "wrapper_context.h", + "wrapper_fault_injection.h", "wrapper_fit.h", "wrapper_gbnf.h", "wrapper_mtmd.h", diff --git a/llama-cpp-wrapper-sources/src/wrapper_sources.rs b/llama-cpp-wrapper-sources/src/wrapper_sources.rs index 06ac02144..86d5cb318 100644 --- a/llama-cpp-wrapper-sources/src/wrapper_sources.rs +++ b/llama-cpp-wrapper-sources/src/wrapper_sources.rs @@ -4,6 +4,7 @@ pub const WRAPPER_SOURCES: &[&str] = &[ "wrapper_chat_templates.cpp", "wrapper_common.cpp", "wrapper_context.cpp", + "wrapper_fault_injection.cpp", "wrapper_fit.cpp", "wrapper_gbnf.cpp", "wrapper_mtmd.cpp", From a671034c5376406ca09726187481d93cdd067d24 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Fri, 30 Oct 2026 19:10:27 +0100 Subject: [PATCH 12/82] Gate the bindings and fault injection crates at full coverage --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4b21a902d..a337fb8cd 100644 --- a/Makefile +++ b/Makefile @@ -76,9 +76,10 @@ coverage: node_modules cargo llvm-cov report ./node_modules/.bin/rust-coverage-check target/llvm-cov.json \ --workspace-root $(CURDIR) \ - --gated llama-cpp-bindings=98 \ + --gated llama-cpp-bindings=100 \ --gated llama-cpp-bindings-types=100 \ --gated llama-cpp-error-recorder=100 \ + --gated llama-cpp-fault-injection=100 \ --gated llama-cpp-ffi-status=100 \ --gated llama-cpp-gbnf=100 \ --gated llama-cpp-log-decoder=100 \ From b63502f97a74aa0d6d80f692a43cd91908f2da58 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Fri, 30 Oct 2026 19:23:20 +0100 Subject: [PATCH 13/82] Pass the harness its command-line arguments so a conflicting thread count is tested, and fold the hf-hub client errors into one --- llama-cpp-test-harness/src/download_model.rs | 6 ++--- .../src/parse_harness_arguments.rs | 4 +-- llama-cpp-test-harness/src/run.rs | 6 +++-- .../src/run_to_conclusions.rs | 27 ++++++++++++++++--- .../tests/harness_self_test.rs | 3 ++- 5 files changed, 34 insertions(+), 12 deletions(-) diff --git a/llama-cpp-test-harness/src/download_model.rs b/llama-cpp-test-harness/src/download_model.rs index 7a194fa90..2fb844fe6 100644 --- a/llama-cpp-test-harness/src/download_model.rs +++ b/llama-cpp-test-harness/src/download_model.rs @@ -7,10 +7,10 @@ use anyhow::Result; /// Returns an error if the HF client cannot be built or the file cannot be downloaded /// (e.g., the repo or file does not exist, or network access fails). pub fn download_model(repo: &str, file: &str) -> Result { - let api = hf_hub::api::sync::ApiBuilder::from_env() + Ok(hf_hub::api::sync::ApiBuilder::from_env() .with_progress(true) - .build()?; - Ok(api.model(repo.to_owned()).get(file)?) + .build() + .and_then(|api| api.model(repo.to_owned()).get(file))?) } #[cfg(test)] diff --git a/llama-cpp-test-harness/src/parse_harness_arguments.rs b/llama-cpp-test-harness/src/parse_harness_arguments.rs index a1d6218da..8f060f7a1 100644 --- a/llama-cpp-test-harness/src/parse_harness_arguments.rs +++ b/llama-cpp-test-harness/src/parse_harness_arguments.rs @@ -17,8 +17,8 @@ fn validate(mut arguments: Arguments) -> Result Result { - validate(Arguments::from_args()) +pub fn parse_harness_arguments(arguments: Arguments) -> Result { + validate(arguments) } #[cfg(test)] diff --git a/llama-cpp-test-harness/src/run.rs b/llama-cpp-test-harness/src/run.rs index 569137630..afb674dc7 100644 --- a/llama-cpp-test-harness/src/run.rs +++ b/llama-cpp-test-harness/src/run.rs @@ -1,5 +1,6 @@ use std::process::ExitCode; +use libtest_mimic::Arguments; use libtest_mimic::Conclusion; use crate::run_to_conclusions::run_to_conclusions; @@ -14,7 +15,7 @@ fn aggregate_exit_code(conclusions: &[Conclusion]) -> ExitCode { #[must_use] pub fn run() -> ExitCode { - match run_to_conclusions() { + match run_to_conclusions(Arguments::from_args()) { Ok(conclusions) => aggregate_exit_code(&conclusions), Err(error) => { eprintln!("llama-cpp-test-harness: {error}"); @@ -27,6 +28,7 @@ pub fn run() -> ExitCode { mod tests { use std::process::ExitCode; + use libtest_mimic::Arguments; use libtest_mimic::Conclusion; use llama_cpp_bindings::error::LlamaCppError; use llama_cpp_bindings::llama_backend::LlamaBackend; @@ -96,7 +98,7 @@ mod tests { .unwrap_or_else(std::sync::PoisonError::into_inner); let _hold = LlamaBackend::init().expect("first init must succeed"); - let outcome = run_to_conclusions(); + let outcome = run_to_conclusions(Arguments::default()); assert_eq!( outcome.err(), diff --git a/llama-cpp-test-harness/src/run_to_conclusions.rs b/llama-cpp-test-harness/src/run_to_conclusions.rs index 0f422b4e3..2debc4849 100644 --- a/llama-cpp-test-harness/src/run_to_conclusions.rs +++ b/llama-cpp-test-harness/src/run_to_conclusions.rs @@ -1,5 +1,6 @@ use std::sync::Arc; +use libtest_mimic::Arguments; use libtest_mimic::Conclusion; use llama_cpp_bindings::llama_backend::LlamaBackend; @@ -12,8 +13,8 @@ use crate::parse_harness_arguments::parse_harness_arguments; /// Returns [`HarnessRunError`] when the CLI arguments conflict with the harness's single-thread /// requirement or the llama backend cannot be initialised. Surfacing these as a typed error keeps /// the failure explicit instead of aborting the process with a panic. -pub fn run_to_conclusions() -> Result, HarnessRunError> { - let arguments = parse_harness_arguments()?; +pub fn run_to_conclusions(arguments: Arguments) -> Result, HarnessRunError> { + let arguments = parse_harness_arguments(arguments)?; let mut backend = LlamaBackend::init()?; let plan = ExecutionPlan::from_inventory(); if plan.requests_void_logs() { @@ -26,18 +27,36 @@ pub fn run_to_conclusions() -> Result, HarnessRunError> { #[cfg(test)] mod tests { + use libtest_mimic::Arguments; + + use crate::harness_arguments_error::HarnessArgumentsError; + use crate::harness_run_error::HarnessRunError; use crate::test_backend_gate::BACKEND_INIT_GATE; use super::run_to_conclusions; + #[test] + fn conflicting_arguments_stop_the_run_before_the_backend_starts() { + assert_eq!( + run_to_conclusions(Arguments { + test_threads: Some(8), + ..Arguments::default() + }) + .err(), + Some(HarnessRunError::ArgumentParsing( + HarnessArgumentsError::ConflictingTestThreads { requested: 8 } + )) + ); + } + #[test] fn empty_inventory_yields_no_conclusions_and_skips_void_logs() { let _gate = BACKEND_INIT_GATE .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - let conclusions = - run_to_conclusions().expect("empty inventory must run without a setup failure"); + let conclusions = run_to_conclusions(Arguments::default()) + .expect("empty inventory must run without a setup failure"); assert!( conclusions.is_empty(), diff --git a/llama-cpp-test-harness/tests/harness_self_test.rs b/llama-cpp-test-harness/tests/harness_self_test.rs index ba026416e..9f07334da 100644 --- a/llama-cpp-test-harness/tests/harness_self_test.rs +++ b/llama-cpp-test-harness/tests/harness_self_test.rs @@ -2,6 +2,7 @@ use std::process::ExitCode; use anyhow::Result; use anyhow::bail; +use libtest_mimic::Arguments; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; use llama_cpp_test_harness::no_op; @@ -137,7 +138,7 @@ const EXPECTED_PASSED: u64 = 6; const EXPECTED_FAILED: u64 = 5; fn main() -> ExitCode { - let conclusions = match run_to_conclusions() { + let conclusions = match run_to_conclusions(Arguments::from_args()) { Ok(conclusions) => conclusions, Err(error) => { eprintln!("harness_self_test: unexpected harness setup failure: {error}"); From 6265a3545c1c450b38a9b73d81cdf0cf41df39b8 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Fri, 30 Oct 2026 19:26:37 +0100 Subject: [PATCH 14/82] Build the system include flags in one place and test the bindgen derive policy --- .../src/bindgen_config.rs | 31 +++++++++++++--- llama-cpp-bindings-build/src/cpp_wrapper.rs | 8 +++-- .../src/compile_commands_file.rs | 25 +++---------- llama-cpp-wrapper-sources/src/lib.rs | 1 + .../src/system_include_flags.rs | 35 +++++++++++++++++++ 5 files changed, 72 insertions(+), 28 deletions(-) create mode 100644 llama-cpp-wrapper-sources/src/system_include_flags.rs diff --git a/llama-cpp-bindings-build/src/bindgen_config.rs b/llama-cpp-bindings-build/src/bindgen_config.rs index e8da81c16..371e9f2fd 100644 --- a/llama-cpp-bindings-build/src/bindgen_config.rs +++ b/llama-cpp-bindings-build/src/bindgen_config.rs @@ -80,6 +80,14 @@ const PRIVATIZED_FIELDS: &[PrivatizedField] = &[ }, ]; +fn derives_for(type_name: &str) -> Vec { + if COMPARED_STRUCTS.contains(&type_name) { + vec!["PartialEq".to_owned()] + } else { + Vec::new() + } +} + #[derive(Clone, Debug)] struct BindingCallbacks { privatized_field_hits: Arc>, @@ -128,11 +136,7 @@ impl bindgen::callbacks::ParseCallbacks for BindingCallbacks { } fn add_derives(&self, info: &bindgen::callbacks::DeriveInfo<'_>) -> Vec { - if COMPARED_STRUCTS.contains(&info.name) { - vec!["PartialEq".to_owned()] - } else { - Vec::new() - } + derives_for(info.name) } fn field_visibility( @@ -248,3 +252,20 @@ fn configure_android_bindgen( builder.clang_arg(format!("--target={target_triple}")) } + +#[cfg(test)] +mod derives_for_tests { + use super::derives_for; + + #[test] + fn only_the_structs_the_bindings_compare_derive_partial_eq() { + assert_eq!( + ["llama_token_data", "llama_logit_bias", "llama_sampler_i"].map(derives_for), + [ + vec!["PartialEq".to_owned()], + vec!["PartialEq".to_owned()], + Vec::new() + ] + ); + } +} diff --git a/llama-cpp-bindings-build/src/cpp_wrapper.rs b/llama-cpp-bindings-build/src/cpp_wrapper.rs index c0ea17f64..20cde9e59 100644 --- a/llama-cpp-bindings-build/src/cpp_wrapper.rs +++ b/llama-cpp-bindings-build/src/cpp_wrapper.rs @@ -1,6 +1,8 @@ +use std::path::Path; + use llama_cpp_wrapper_sources::cpp_standard::CPP_STANDARD; use llama_cpp_wrapper_sources::fault_injection_define::FAULT_INJECTION_DEFINE; -use llama_cpp_wrapper_sources::vendored_include_dirs::VENDORED_INCLUDE_DIRS; +use llama_cpp_wrapper_sources::system_include_flags::system_include_flags; use llama_cpp_wrapper_sources::wrapper_include_dirs::WRAPPER_INCLUDE_DIRS; use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; @@ -16,8 +18,8 @@ pub fn compile_cpp_wrappers(target_os: TargetOs) -> Result<(), BuildError> { build.include(include_dir); } - for include_dir in VENDORED_INCLUDE_DIRS { - build.flag(format!("-isystem{include_dir}")); + for system_include_flag in system_include_flags(Path::new(".")) { + build.flag(system_include_flag); } build.flag_if_supported(format!("-std={CPP_STANDARD}")); diff --git a/llama-cpp-wrapper-sources/src/compile_commands_file.rs b/llama-cpp-wrapper-sources/src/compile_commands_file.rs index 380aec0c2..a4ffb3374 100644 --- a/llama-cpp-wrapper-sources/src/compile_commands_file.rs +++ b/llama-cpp-wrapper-sources/src/compile_commands_file.rs @@ -5,7 +5,7 @@ use std::path::PathBuf; use crate::compile_command::CompileCommand; use crate::cpp_standard::CPP_STANDARD; use crate::fault_injection_define::FAULT_INJECTION_DEFINE; -use crate::vendored_include_dirs::VENDORED_INCLUDE_DIRS; +use crate::system_include_flags::system_include_flags; use crate::wrapper_include_dirs::WRAPPER_INCLUDE_DIRS; use crate::wrapper_source_paths::wrapper_source_paths; use crate::wrapper_sources_error::WrapperSourcesError; @@ -37,12 +37,7 @@ impl CompileCommandsFile { arguments.push(format!("-I{}", absolute_path(&self.sys_dir, include_dir))); } - for include_dir in VENDORED_INCLUDE_DIRS { - arguments.push(format!( - "-isystem{}", - absolute_path(&self.sys_dir, include_dir) - )); - } + arguments.extend(system_include_flags(&self.sys_dir)); arguments.push("-c".to_owned()); arguments.push(file.display().to_string()); @@ -82,7 +77,7 @@ mod tests { use std::path::PathBuf; use super::CompileCommandsFile; - use crate::vendored_include_dirs::VENDORED_INCLUDE_DIRS; + use crate::system_include_flags::system_include_flags; use crate::wrapper_include_dirs::WRAPPER_INCLUDE_DIRS; use crate::wrapper_source_paths::wrapper_source_paths; use crate::wrapper_sources_error::WrapperSourcesError; @@ -136,7 +131,7 @@ mod tests { } #[test] - fn every_vendored_include_dir_is_a_system_include_resolved_against_the_sys_dir() { + fn every_command_includes_the_vendored_headers_as_system_headers() { let commands = database_for("/unused").commands(); assert_eq!( @@ -149,17 +144,7 @@ mod tests { .cloned() .collect::>()) .unwrap_or_default(), - VENDORED_INCLUDE_DIRS - .iter() - .map(|include_dir| { - format!( - "-isystem{}", - Path::new("/repo/llama-cpp-bindings-sys") - .join(include_dir) - .display() - ) - }) - .collect::>() + system_include_flags(Path::new("/repo/llama-cpp-bindings-sys")) ); } diff --git a/llama-cpp-wrapper-sources/src/lib.rs b/llama-cpp-wrapper-sources/src/lib.rs index 996af00cd..d7aa2ce27 100644 --- a/llama-cpp-wrapper-sources/src/lib.rs +++ b/llama-cpp-wrapper-sources/src/lib.rs @@ -12,6 +12,7 @@ pub mod compile_command; pub mod compile_commands_file; pub mod cpp_standard; pub mod fault_injection_define; +pub mod system_include_flags; pub mod vendored_include_dirs; pub mod wrapper_headers; pub mod wrapper_include_dirs; diff --git a/llama-cpp-wrapper-sources/src/system_include_flags.rs b/llama-cpp-wrapper-sources/src/system_include_flags.rs new file mode 100644 index 000000000..1a227a65b --- /dev/null +++ b/llama-cpp-wrapper-sources/src/system_include_flags.rs @@ -0,0 +1,35 @@ +use std::path::Path; + +use crate::vendored_include_dirs::VENDORED_INCLUDE_DIRS; + +/// The compiler flags that include every vendored header directory under `sys_dir` as a system +/// directory, so warnings in vendored headers do not surface while ours still do. +#[must_use] +pub fn system_include_flags(sys_dir: &Path) -> Vec { + VENDORED_INCLUDE_DIRS + .iter() + .map(|include_dir| format!("-isystem{}", sys_dir.join(include_dir).display())) + .collect() +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use super::system_include_flags; + + #[test] + fn every_vendored_include_dir_becomes_a_system_include_under_the_sys_dir() { + assert_eq!( + system_include_flags(Path::new("/repo/llama-cpp-bindings-sys")), + [ + "-isystem/repo/llama-cpp-bindings-sys/GSL/include", + "-isystem/repo/llama-cpp-bindings-sys/llama.cpp", + "-isystem/repo/llama-cpp-bindings-sys/llama.cpp/common", + "-isystem/repo/llama-cpp-bindings-sys/llama.cpp/include", + "-isystem/repo/llama-cpp-bindings-sys/llama.cpp/ggml/include", + "-isystem/repo/llama-cpp-bindings-sys/llama.cpp/vendor", + ] + ); + } +} From e90d76ee49179fb618c52657144397156c38eb27 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Fri, 30 Oct 2026 19:39:02 +0100 Subject: [PATCH 15/82] Gate the test harness at full coverage --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a337fb8cd..a7d181ffd 100644 --- a/Makefile +++ b/Makefile @@ -83,7 +83,7 @@ coverage: node_modules --gated llama-cpp-ffi-status=100 \ --gated llama-cpp-gbnf=100 \ --gated llama-cpp-log-decoder=100 \ - --gated llama-cpp-test-harness=99 \ + --gated llama-cpp-test-harness=100 \ --gated llama-cpp-test-harness-macros=100 \ --gated llama-cpp-wrapper-sources=100 From a0b70cd91eddf999e181406b04d1bbf056cb6dff Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Fri, 30 Oct 2026 20:01:33 +0100 Subject: [PATCH 16/82] Give test-only error paths eager context instead of unexecuted closures and gate the tests crate at full coverage --- Makefile | 1 + .../tests/completion_usage_accounting.rs | 2 +- .../tests/embedding_generation.rs | 2 +- .../tests/grammar_sampler.rs | 6 ++-- .../json_schema_constrained_generation.rs | 4 +-- .../tests/kv_cache_memory_availability.rs | 4 +-- .../tests/kv_cache_position_bounds.rs | 6 ++-- ...cache_position_shift_rope_compatibility.rs | 6 ++-- .../tests/llguidance_sampler.rs | 3 +- .../tests/multimodal_chunk_copy.rs | 6 ++-- .../tests/multimodal_chunk_encoding.rs | 4 +-- .../tests/multimodal_chunk_evaluation.rs | 5 ++-- .../tests/multimodal_chunk_tokenization.rs | 28 ++++++------------- .../tests/multimodal_fusion.rs | 4 +-- .../multimodal_projector_capabilities.rs | 4 +-- .../multimodal_prompt_parts_tokenization.rs | 6 ++-- .../multimodal_reasoning_classification.rs | 10 +++---- .../tests/multimodal_usage_accounting.rs | 10 +++---- .../tests/parameter_fitting.rs | 5 ++-- 19 files changed, 51 insertions(+), 65 deletions(-) diff --git a/Makefile b/Makefile index a7d181ffd..da2254488 100644 --- a/Makefile +++ b/Makefile @@ -77,6 +77,7 @@ coverage: node_modules ./node_modules/.bin/rust-coverage-check target/llvm-cov.json \ --workspace-root $(CURDIR) \ --gated llama-cpp-bindings=100 \ + --gated llama-cpp-bindings-tests=100 \ --gated llama-cpp-bindings-types=100 \ --gated llama-cpp-error-recorder=100 \ --gated llama-cpp-fault-injection=100 \ diff --git a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs index c8c6e92e1..f2c50c0f9 100644 --- a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs +++ b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs @@ -72,7 +72,7 @@ fn raw_prompt_completion_usage_matches_the_classified_tokens( let mut classifier = SampledTokenClassifier::undeterminable(model); let tokens_list = model .str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always) - .with_context(|| format!("failed to tokenize {prompt}"))?; + .context(format!("failed to tokenize {prompt}"))?; let prompt_token_count = u64::try_from(tokens_list.len())?; let mut decoder = encoding_rs::UTF_8.new_decoder(); diff --git a/llama-cpp-bindings-tests/tests/embedding_generation.rs b/llama-cpp-bindings-tests/tests/embedding_generation.rs index 9f3846978..60f1b96af 100644 --- a/llama-cpp-bindings-tests/tests/embedding_generation.rs +++ b/llama-cpp-bindings-tests/tests/embedding_generation.rs @@ -37,7 +37,7 @@ fn embedding_generation_produces_vectors(fixture: &LlamaFixture<'_>) -> Result<( let prompt = "Hello my name is"; let tokens = model .str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always) - .with_context(|| format!("failed to tokenize {prompt}"))?; + .context(format!("failed to tokenize {prompt}"))?; let prompt_token_count = u64::try_from(tokens.len())?; let n_ctx = usize::try_from(ctx.n_ctx())?; diff --git a/llama-cpp-bindings-tests/tests/grammar_sampler.rs b/llama-cpp-bindings-tests/tests/grammar_sampler.rs index c5b2cf3f6..13abdb38a 100644 --- a/llama-cpp-bindings-tests/tests/grammar_sampler.rs +++ b/llama-cpp-bindings-tests/tests/grammar_sampler.rs @@ -1,7 +1,7 @@ use std::ffi::CString; +use anyhow::Context as _; use anyhow::Result; -use anyhow::anyhow; use anyhow::bail; use llama_cpp_bindings::BareJsonToolCalls; use llama_cpp_bindings::GbnfValidationError; @@ -96,10 +96,10 @@ fn grammar_sampler_constrains_output_to_yes_or_no(fixture: &LlamaFixture<'_>) -> let first_char = piece .chars() .next() - .ok_or_else(|| anyhow!("piece should have at least one character"))? + .context("piece should have at least one character")? .to_lowercase() .next() - .ok_or_else(|| anyhow!("lowercase iterator should yield a character"))?; + .context("lowercase iterator should yield a character")?; assert!( first_char == 'y' || first_char == 'n', diff --git a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs index ddab5491f..9feaaf5b1 100644 --- a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs +++ b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs @@ -1,7 +1,7 @@ use std::io::Write; +use anyhow::Context as _; use anyhow::Result; -use anyhow::anyhow; use llama_cpp_bindings::SampledToken; use llama_cpp_bindings::SampledTokenClassifier; use llama_cpp_bindings::context::LlamaContext; @@ -204,7 +204,7 @@ fn json_schema_constrains_output(fixture: &LlamaFixture<'_>) -> Result<()> { let parsed = serde_json::Deserializer::from_str(&generated) .into_iter::() .next() - .ok_or_else(|| anyhow!("model produced no JSON value"))??; + .context("model produced no JSON value")??; assert!(parsed.get("city").is_some()); assert!(parsed.get("temperature").is_some()); diff --git a/llama-cpp-bindings-tests/tests/kv_cache_memory_availability.rs b/llama-cpp-bindings-tests/tests/kv_cache_memory_availability.rs index a19d961c4..62607c5e3 100644 --- a/llama-cpp-bindings-tests/tests/kv_cache_memory_availability.rs +++ b/llama-cpp-bindings-tests/tests/kv_cache_memory_availability.rs @@ -1,7 +1,7 @@ use std::num::NonZeroU8; +use anyhow::Context as _; use anyhow::Result; -use anyhow::anyhow; use llama_cpp_bindings::ClearKvCacheSeqError; use llama_cpp_bindings::CopyKvCacheSeqError; use llama_cpp_bindings::KvCacheSeqAddError; @@ -67,7 +67,7 @@ fn kv_cache_operations_respect_an_embedding_context_without_memory( context.kv_cache_seq_add(0, None, None, 1), Err(KvCacheSeqAddError::MemoryHandleUnavailable) ); - let divisor = NonZeroU8::new(2).ok_or_else(|| anyhow!("2 is non-zero"))?; + let divisor = NonZeroU8::new(2).context("2 is non-zero")?; assert_eq!( context.kv_cache_seq_div(0, None, None, divisor), Err(KvCacheSeqDivError::MemoryHandleUnavailable) diff --git a/llama-cpp-bindings-tests/tests/kv_cache_position_bounds.rs b/llama-cpp-bindings-tests/tests/kv_cache_position_bounds.rs index afa35c52b..f3de9b6f1 100644 --- a/llama-cpp-bindings-tests/tests/kv_cache_position_bounds.rs +++ b/llama-cpp-bindings-tests/tests/kv_cache_position_bounds.rs @@ -1,7 +1,7 @@ use std::num::NonZeroU8; +use anyhow::Context as _; use anyhow::Result; -use anyhow::anyhow; use llama_cpp_bindings::error::ClearKvCacheSeqError; use llama_cpp_bindings::error::CopyKvCacheSeqError; use llama_cpp_bindings::error::KvCacheSeqAddError; @@ -326,7 +326,7 @@ fn kv_cache_seq_add_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> fn kv_cache_seq_div_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; - let divisor = NonZeroU8::new(2).ok_or_else(|| anyhow!("2 is non-zero"))?; + let divisor = NonZeroU8::new(2).context("2 is non-zero")?; let result = context.kv_cache_seq_div(0, Some(u32::MAX), None, divisor); assert_eq!( @@ -374,7 +374,7 @@ fn kv_cache_seq_div_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> fn kv_cache_seq_div_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; - let divisor = NonZeroU8::new(2).ok_or_else(|| anyhow!("2 is non-zero"))?; + let divisor = NonZeroU8::new(2).context("2 is non-zero")?; let result = context.kv_cache_seq_div(0, Some(0), Some(u32::MAX), divisor); assert_eq!( diff --git a/llama-cpp-bindings-tests/tests/kv_cache_position_shift_rope_compatibility.rs b/llama-cpp-bindings-tests/tests/kv_cache_position_shift_rope_compatibility.rs index 9d9c81e7a..631bf7f29 100644 --- a/llama-cpp-bindings-tests/tests/kv_cache_position_shift_rope_compatibility.rs +++ b/llama-cpp-bindings-tests/tests/kv_cache_position_shift_rope_compatibility.rs @@ -1,7 +1,7 @@ use std::num::NonZeroU8; +use anyhow::Context as _; use anyhow::Result; -use anyhow::anyhow; use llama_cpp_bindings::error::KvCacheSeqAddError; use llama_cpp_bindings::error::KvCacheSeqDivError; use llama_cpp_bindings_tests::prime_kv_cache::prime_kv_cache; @@ -60,7 +60,7 @@ fn kv_cache_seq_div_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> prime_kv_cache(fixture, &mut context)?; - let divisor = NonZeroU8::new(2).ok_or_else(|| anyhow!("2 is non-zero"))?; + let divisor = NonZeroU8::new(2).context("2 is non-zero")?; let result = context.kv_cache_seq_div(0, Some(0), None, divisor); assert_eq!( @@ -104,7 +104,7 @@ fn kv_cache_seq_div_succeeds_on_embedding_model(fixture: &LlamaFixture<'_>) -> R prime_kv_cache(fixture, &mut context)?; - let divisor = NonZeroU8::new(2).ok_or_else(|| anyhow!("2 is non-zero"))?; + let divisor = NonZeroU8::new(2).context("2 is non-zero")?; let result = context.kv_cache_seq_div(0, Some(0), None, divisor); assert!(result.is_ok()); diff --git a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs index ca5bf01ee..f0af41d09 100644 --- a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs +++ b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs @@ -2,7 +2,6 @@ use std::ffi::CStr; use anyhow::Context as _; use anyhow::Result; -use anyhow::anyhow; use llama_cpp_bindings::GrammarError; use llama_cpp_bindings::SampleError; use llama_cpp_bindings::SamplerAcceptError; @@ -470,7 +469,7 @@ fn reset_rolls_back_accepted_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { .str_to_token("yes", AddBos::Never, ParseSpecialTokens::Always)?; let first_allowed_token = *yes_tokens .first() - .ok_or_else(|| anyhow!("the tokenizer must produce a token for \"yes\""))?; + .context("the tokenizer must produce a token for \"yes\"")?; sampler .accept(first_allowed_token) diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs index c2d2bb8d9..8e426de92 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs @@ -1,5 +1,5 @@ +use anyhow::Context as _; use anyhow::Result; -use anyhow::anyhow; use llama_cpp_bindings::mtmd::MtmdBitmap; use llama_cpp_bindings::mtmd::MtmdInputText; use llama_cpp_test_harness::LlamaFixture; @@ -35,9 +35,7 @@ fn copy_creates_owned_duplicate(fixture: &LlamaFixture<'_>) -> Result<()> { parse_special: true, }; let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - let first_chunk = chunks - .get(0) - .ok_or_else(|| anyhow!("missing first chunk"))?; + let first_chunk = chunks.get(0).context("missing first chunk")?; let copied = first_chunk.copy()?; assert!(copied.owned); diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs index 003ee0aa4..f41de20a7 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs @@ -1,5 +1,5 @@ +use anyhow::Context as _; use anyhow::Result; -use anyhow::anyhow; use llama_cpp_bindings::mtmd::MtmdBitmap; use llama_cpp_bindings::mtmd::MtmdInputChunkType; use llama_cpp_bindings::mtmd::MtmdInputText; @@ -40,7 +40,7 @@ fn encode_chunk_succeeds_for_image_chunk(fixture: &LlamaFixture<'_>) -> Result<( for chunk_index in 0..chunks.len() { let chunk = chunks .get(chunk_index) - .ok_or_else(|| anyhow!("missing chunk at index {chunk_index}"))?; + .context(format!("missing chunk at index {chunk_index}"))?; if chunk.chunk_type() == Ok(MtmdInputChunkType::Image) { let result = mtmd_ctx.encode_chunk(&chunk); assert!(result.is_ok()); diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs index c52488745..a0ab644a1 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs @@ -1,6 +1,5 @@ use anyhow::Context; use anyhow::Result; -use anyhow::anyhow; use anyhow::bail; use llama_cpp_bindings::EvalMultimodalChunksParams; use llama_cpp_bindings::FfiContractError; @@ -455,7 +454,7 @@ fn eval_chunks_succeeds_for_a_standard_image(fixture: &LlamaFixture<'_>) -> Resu let image_path = fixtures.join("llamas.jpg"); let image_path_str = image_path .to_str() - .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; + .context("image path is not valid UTF-8")?; let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; let input_text = MtmdInputText { text: "What is in this image? <__media__>".to_string(), @@ -545,7 +544,7 @@ fn eval_chunks_supports_extreme_dimensions(fixture: &LlamaFixture<'_>) -> Result for (width, height) in extreme_dimensions { eval_synthetic_bitmap(fixture, width, height) - .with_context(|| format!("dimension {width}x{height} should succeed"))?; + .context(format!("dimension {width}x{height} should succeed"))?; } Ok(()) diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs index 4768eebc2..e5ef6abdf 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs @@ -1,5 +1,5 @@ +use anyhow::Context as _; use anyhow::Result; -use anyhow::anyhow; use llama_cpp_bindings::mtmd::MtmdBitmap; use llama_cpp_bindings::mtmd::MtmdInputChunkType; use llama_cpp_bindings::mtmd::MtmdInputChunksError; @@ -33,9 +33,7 @@ use crate::tokenize_synthetic::tokenize_synthetic; )] fn text_chunk_has_text_type(fixture: &LlamaFixture<'_>) -> Result<()> { let chunks = tokenize_synthetic(fixture, "Hello world <__media__>")?; - let first_chunk = chunks - .get(0) - .ok_or_else(|| anyhow!("missing first chunk"))?; + let first_chunk = chunks.get(0).context("missing first chunk")?; assert_eq!(first_chunk.chunk_type()?, MtmdInputChunkType::Text); Ok(()) } @@ -60,12 +58,10 @@ fn text_chunk_has_text_type(fixture: &LlamaFixture<'_>) -> Result<()> { )] fn text_chunk_returns_text_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { let chunks = tokenize_synthetic(fixture, "Hello world <__media__>")?; - let first_chunk = chunks - .get(0) - .ok_or_else(|| anyhow!("missing first chunk"))?; + let first_chunk = chunks.get(0).context("missing first chunk")?; let tokens = first_chunk .text_tokens_of(first_chunk.chunk_type()?) - .ok_or_else(|| anyhow!("the first chunk of a text prompt must carry tokens"))?; + .context("the first chunk of a text prompt must carry tokens")?; assert_ne!(tokens, Vec::::new()); @@ -92,9 +88,7 @@ fn text_chunk_returns_text_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { )] fn chunk_n_tokens_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { let chunks = tokenize_synthetic(fixture, "Hello world <__media__>")?; - let first_chunk = chunks - .get(0) - .ok_or_else(|| anyhow!("missing first chunk"))?; + let first_chunk = chunks.get(0).context("missing first chunk")?; assert!(first_chunk.n_tokens() > 0); Ok(()) } @@ -119,9 +113,7 @@ fn chunk_n_tokens_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { )] fn chunk_n_positions_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { let chunks = tokenize_synthetic(fixture, "Hello world <__media__>")?; - let first_chunk = chunks - .get(0) - .ok_or_else(|| anyhow!("missing first chunk"))?; + let first_chunk = chunks.get(0).context("missing first chunk")?; assert!(first_chunk.n_positions() > 0); Ok(()) } @@ -146,9 +138,7 @@ fn chunk_n_positions_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { )] fn text_chunk_id_returns_none(fixture: &LlamaFixture<'_>) -> Result<()> { let chunks = tokenize_synthetic(fixture, "Hello <__media__>")?; - let first_chunk = chunks - .get(0) - .ok_or_else(|| anyhow!("missing first chunk"))?; + let first_chunk = chunks.get(0).context("missing first chunk")?; assert_eq!(first_chunk.chunk_type()?, MtmdInputChunkType::Text); assert!(first_chunk.id().is_none()); Ok(()) @@ -177,7 +167,7 @@ fn image_chunk_returns_none_for_text_tokens(fixture: &LlamaFixture<'_>) -> Resul for chunk_index in 0..chunks.len() { let chunk = chunks .get(chunk_index) - .ok_or_else(|| anyhow!("missing chunk at index {chunk_index}"))?; + .context(format!("missing chunk at index {chunk_index}"))?; if chunk.chunk_type()? == MtmdInputChunkType::Image { assert_eq!(chunk.text_tokens_of(MtmdInputChunkType::Image), None); @@ -210,7 +200,7 @@ fn image_chunk_id_returns_some(fixture: &LlamaFixture<'_>) -> Result<()> { for chunk_index in 0..chunks.len() { let chunk = chunks .get(chunk_index) - .ok_or_else(|| anyhow!("missing chunk at index {chunk_index}"))?; + .context(format!("missing chunk at index {chunk_index}"))?; if chunk.chunk_type() == Ok(MtmdInputChunkType::Image) { assert!(chunk.id().is_some()); return Ok(()); diff --git a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs index b459dd012..bf1118a13 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs @@ -52,9 +52,9 @@ fn load_fixture_bitmap(mtmd_ctx: &MtmdContext, file_name: &str) -> Result for chunk_index in 0..chunks.len() { let chunk = chunks .get(chunk_index) - .ok_or_else(|| anyhow!("missing chunk at index {chunk_index}"))?; + .context(format!("missing chunk at index {chunk_index}"))?; if chunk.chunk_type() == Ok(MtmdInputChunkType::Image) { let value = mtmd_ctx.decode_use_non_causal(&chunk); let printed = format!("{value:?}"); diff --git a/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs b/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs index 08e3ea8f6..88ac48d88 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs @@ -1,7 +1,7 @@ use std::ffi::CString; +use anyhow::Context as _; use anyhow::Result; -use anyhow::anyhow; use llama_cpp_bindings::model::AddBos; use llama_cpp_bindings::model::ParseSpecialTokens; use llama_cpp_bindings::mtmd::MtmdBitmap; @@ -33,7 +33,7 @@ fn tokens_of_text_chunks(chunks: &MtmdInputChunks) -> Result> { for chunk_index in 0..chunks.len() { let chunk = chunks .get(chunk_index) - .ok_or_else(|| anyhow!("missing chunk at index {chunk_index}"))?; + .context(format!("missing chunk at index {chunk_index}"))?; if let Some(chunk_tokens) = chunk.text_tokens_of(chunk.chunk_type()?) { tokens.extend_from_slice(chunk_tokens); @@ -74,7 +74,7 @@ fn only_bitmap_parts_become_image_chunks(fixture: &LlamaFixture<'_>) -> Result<( for chunk_index in 0..chunks.len() { let chunk = chunks .get(chunk_index) - .ok_or_else(|| anyhow!("missing chunk at index {chunk_index}"))?; + .context(format!("missing chunk at index {chunk_index}"))?; if chunk.chunk_type()? == MtmdInputChunkType::Image { image_chunks += 1; diff --git a/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs index 0c6a2f329..6189c76d2 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs @@ -1,5 +1,5 @@ +use anyhow::Context as _; use anyhow::Result; -use anyhow::anyhow; use anyhow::bail; use llama_cpp_bindings::BareJsonToolCalls; use llama_cpp_bindings::ChatAssistantTurn; @@ -52,7 +52,7 @@ fn gemma4_classifier_emits_reasoning_for_multimodal_thinking_prompt( let image_path = fixtures_dir().join("llamas.jpg"); let image_path_str = image_path .to_str() - .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; + .context("image path is not valid UTF-8")?; let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; let session = media_question_chat_session(model, PROMPT_QUESTION)?; @@ -144,7 +144,7 @@ fn mistral3_classifier_emits_reasoning_for_multimodal_thinking_prompt( let image_path = fixtures_dir().join("llamas.jpg"); let image_path_str = image_path .to_str() - .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; + .context("image path is not valid UTF-8")?; let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; let marker = mtmd_default_marker()?; @@ -263,7 +263,7 @@ fn qwen35_classifier_emits_reasoning_for_multimodal_thinking_prompt( let image_path = fixtures_dir().join("llamas.jpg"); let image_path_str = image_path .to_str() - .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; + .context("image path is not valid UTF-8")?; let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; let session = media_question_chat_session(model, PROMPT_QUESTION)?; @@ -355,7 +355,7 @@ fn qwen36_classifier_emits_reasoning_for_multimodal_thinking_prompt( let image_path = fixtures_dir().join("llamas.jpg"); let image_path_str = image_path .to_str() - .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; + .context("image path is not valid UTF-8")?; let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; let session = media_question_chat_session(model, PROMPT_QUESTION)?; diff --git a/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs b/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs index 52f951562..bbc086d48 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs @@ -1,5 +1,5 @@ +use anyhow::Context as _; use anyhow::Result; -use anyhow::anyhow; use anyhow::bail; use llama_cpp_bindings::EvalMultimodalChunksParams; use llama_cpp_bindings::TokenUsage; @@ -28,7 +28,7 @@ fn build_multimodal_chunks_and_eval_into_usage( let image_path = fixtures_dir().join("llamas.jpg"); let image_path_str = image_path .to_str() - .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; + .context("image path is not valid UTF-8")?; let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; let marker = mtmd_default_marker()?; @@ -194,7 +194,7 @@ fn text_chunk_records_prompt_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { let text_chunk = (0..fitted_chunks.len()) .filter_map(|index| FittedMtmdInputChunk::at(&fitted_chunks, index)) .find(|chunk| chunk.chunk_type() == MtmdInputChunkType::Text) - .ok_or_else(|| anyhow!("text-only tokenization should produce at least one text chunk"))?; + .context("text-only tokenization should produce at least one text chunk")?; let n_tokens = u64::try_from(text_chunk.chunk().n_tokens())?; @@ -243,7 +243,7 @@ fn image_chunk_records_input_image_tokens_only(fixture: &LlamaFixture<'_>) -> Re let image_path = fixtures_dir().join("llamas.jpg"); let image_path_str = image_path .to_str() - .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; + .context("image path is not valid UTF-8")?; let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; let marker = mtmd_default_marker()?; @@ -266,7 +266,7 @@ fn image_chunk_records_input_image_tokens_only(fixture: &LlamaFixture<'_>) -> Re let image_chunk = (0..fitted_chunks.len()) .filter_map(|index| FittedMtmdInputChunk::at(&fitted_chunks, index)) .find(|chunk| chunk.chunk_type() == MtmdInputChunkType::Image) - .ok_or_else(|| anyhow!("multimodal tokenization should produce an image chunk"))?; + .context("multimodal tokenization should produce an image chunk")?; let n_tokens = u64::try_from(image_chunk.chunk().n_tokens())?; if n_tokens == 0 { diff --git a/llama-cpp-bindings-tests/tests/parameter_fitting.rs b/llama-cpp-bindings-tests/tests/parameter_fitting.rs index 8e600d1f0..5d778c9ac 100644 --- a/llama-cpp-bindings-tests/tests/parameter_fitting.rs +++ b/llama-cpp-bindings-tests/tests/parameter_fitting.rs @@ -3,7 +3,6 @@ use std::num::NonZeroU32; use std::pin::pin; use anyhow::Result; -use anyhow::anyhow; use llama_cpp_bindings::context::params::LlamaContextParams; use llama_cpp_bindings::max_devices; use llama_cpp_bindings::model::params::LlamaModelParams; @@ -62,7 +61,7 @@ fn fit_params_succeeds_with_test_model(fixture: &LlamaFixture<'_>) -> Result<()> llama_cpp_bindings_sys::GGML_LOG_LEVEL_NONE, ); - let fit = result.map_err(|fit_error| anyhow!("fit_params failed: {fit_error:?}"))?; + let fit = result?; assert!(fit.n_ctx > 0); Ok(()) @@ -105,7 +104,7 @@ fn fit_params_aligns_the_extra_model_context_with_the_fitted_one( llama_cpp_bindings_sys::GGML_LOG_LEVEL_NONE, ); - let fit = result.map_err(|fit_error| anyhow!("fit_params failed: {fit_error:?}"))?; + let fit = result?; assert!(fit.n_ctx > 0); assert_ne!( extra_context_params.n_ctx(), From a4d482c4bd5c7cb1847ccfc2f25a76bc7a6a5574 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Fri, 30 Oct 2026 21:35:50 +0100 Subject: [PATCH 17/82] Plan the native build as tested data and gate the build crate at full coverage --- Cargo.lock | 1 + Makefile | 1 + llama-cpp-bindings-build/Cargo.toml | 1 + .../build/cmake/android.toolchain.cmake | 0 .../lib/clang/20/include/stddef.h | 0 .../fixtures/bindings/llama_types.h | 15 + .../bindings/missing_privatized_field.h | 5 + .../fixtures/bindings/wrapper.h | 8 + .../fixtures/cmake_project/CMakeLists.txt | 10 + .../fixtures/cmake_project/fixture.c | 3 + .../fixtures/wrapper/vendor/vendored.h | 3 + .../fixtures/wrapper/wrapper_fixture.cpp | 9 + .../src/android_architecture.rs | 69 ++ llama-cpp-bindings-build/src/android_ndk.rs | 627 ++++++++--------- llama-cpp-bindings-build/src/apple_variant.rs | 5 - .../src/bindgen_config.rs | 271 ------- .../src/binding_callbacks.rs | 123 ++++ .../src/bindings_generation.rs | 295 ++++++++ llama-cpp-bindings-build/src/build_context.rs | 255 +++++++ .../src/build_environment.rs | 184 +++++ llama-cpp-bindings-build/src/build_error.rs | 69 ++ llama-cpp-bindings-build/src/build_feature.rs | 20 + .../src/cargo_directive.rs | 99 +++ llama-cpp-bindings-build/src/cmake_build.rs | 60 ++ llama-cpp-bindings-build/src/cmake_config.rs | 368 ---------- .../src/cmake_configuration.rs | 525 ++++++++++++++ llama-cpp-bindings-build/src/cmake_define.rs | 22 + llama-cpp-bindings-build/src/cpp_wrapper.rs | 42 -- llama-cpp-bindings-build/src/cpu_tuning.rs | 294 ++++++++ .../src/emit_directives.rs | 73 ++ .../src/enabled_build_features.rs | 81 +++ llama-cpp-bindings-build/src/host_os.rs | 72 ++ llama-cpp-bindings-build/src/lib.rs | 274 +++----- .../src/library_linking.rs | 381 ---------- .../src/link_directives.rs | 663 ++++++++++++++++++ llama-cpp-bindings-build/src/link_kind.rs | 7 + llama-cpp-bindings-build/src/native_build.rs | 243 +++++++ .../src/native_library.rs | 45 +- .../src/privatized_field.rs | 20 + .../src/rebuild_tracking.rs | 15 - .../src/rebuild_triggers.rs | 65 ++ llama-cpp-bindings-build/src/target_os.rs | 146 ++-- .../src/wrapper_compilation.rs | 158 +++++ llama-cpp-bindings-sys/build.rs | 7 +- 44 files changed, 4024 insertions(+), 1610 deletions(-) create mode 100644 llama-cpp-bindings-build/fixtures/android_ndk/build/cmake/android.toolchain.cmake create mode 100644 llama-cpp-bindings-build/fixtures/android_ndk/toolchains/llvm/prebuilt/linux-x86_64/lib/clang/20/include/stddef.h create mode 100644 llama-cpp-bindings-build/fixtures/bindings/llama_types.h create mode 100644 llama-cpp-bindings-build/fixtures/bindings/missing_privatized_field.h create mode 100644 llama-cpp-bindings-build/fixtures/bindings/wrapper.h create mode 100644 llama-cpp-bindings-build/fixtures/cmake_project/CMakeLists.txt create mode 100644 llama-cpp-bindings-build/fixtures/cmake_project/fixture.c create mode 100644 llama-cpp-bindings-build/fixtures/wrapper/vendor/vendored.h create mode 100644 llama-cpp-bindings-build/fixtures/wrapper/wrapper_fixture.cpp create mode 100644 llama-cpp-bindings-build/src/android_architecture.rs delete mode 100644 llama-cpp-bindings-build/src/apple_variant.rs delete mode 100644 llama-cpp-bindings-build/src/bindgen_config.rs create mode 100644 llama-cpp-bindings-build/src/binding_callbacks.rs create mode 100644 llama-cpp-bindings-build/src/bindings_generation.rs create mode 100644 llama-cpp-bindings-build/src/build_context.rs create mode 100644 llama-cpp-bindings-build/src/build_environment.rs create mode 100644 llama-cpp-bindings-build/src/build_error.rs create mode 100644 llama-cpp-bindings-build/src/build_feature.rs create mode 100644 llama-cpp-bindings-build/src/cargo_directive.rs create mode 100644 llama-cpp-bindings-build/src/cmake_build.rs delete mode 100644 llama-cpp-bindings-build/src/cmake_config.rs create mode 100644 llama-cpp-bindings-build/src/cmake_configuration.rs create mode 100644 llama-cpp-bindings-build/src/cmake_define.rs delete mode 100644 llama-cpp-bindings-build/src/cpp_wrapper.rs create mode 100644 llama-cpp-bindings-build/src/cpu_tuning.rs create mode 100644 llama-cpp-bindings-build/src/emit_directives.rs create mode 100644 llama-cpp-bindings-build/src/enabled_build_features.rs create mode 100644 llama-cpp-bindings-build/src/host_os.rs delete mode 100644 llama-cpp-bindings-build/src/library_linking.rs create mode 100644 llama-cpp-bindings-build/src/link_directives.rs create mode 100644 llama-cpp-bindings-build/src/link_kind.rs create mode 100644 llama-cpp-bindings-build/src/native_build.rs create mode 100644 llama-cpp-bindings-build/src/privatized_field.rs delete mode 100644 llama-cpp-bindings-build/src/rebuild_tracking.rs create mode 100644 llama-cpp-bindings-build/src/rebuild_triggers.rs create mode 100644 llama-cpp-bindings-build/src/wrapper_compilation.rs diff --git a/Cargo.lock b/Cargo.lock index 3b8ec34cd..fbdabb0d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1194,6 +1194,7 @@ dependencies = [ "bindgen", "cc", "cmake", + "enumflags2", "find_cuda_helper", "llama-cpp-wrapper-sources", "thiserror", diff --git a/Makefile b/Makefile index da2254488..7d65ee485 100644 --- a/Makefile +++ b/Makefile @@ -77,6 +77,7 @@ coverage: node_modules ./node_modules/.bin/rust-coverage-check target/llvm-cov.json \ --workspace-root $(CURDIR) \ --gated llama-cpp-bindings=100 \ + --gated llama-cpp-bindings-build=100 \ --gated llama-cpp-bindings-tests=100 \ --gated llama-cpp-bindings-types=100 \ --gated llama-cpp-error-recorder=100 \ diff --git a/llama-cpp-bindings-build/Cargo.toml b/llama-cpp-bindings-build/Cargo.toml index f69f99f99..4bfc4d5a1 100644 --- a/llama-cpp-bindings-build/Cargo.toml +++ b/llama-cpp-bindings-build/Cargo.toml @@ -10,6 +10,7 @@ repository.workspace = true bindgen = { workspace = true } cc = { workspace = true } cmake = { workspace = true } +enumflags2 = { workspace = true } find_cuda_helper = { workspace = true } llama-cpp-wrapper-sources = { workspace = true } thiserror = { workspace = true } diff --git a/llama-cpp-bindings-build/fixtures/android_ndk/build/cmake/android.toolchain.cmake b/llama-cpp-bindings-build/fixtures/android_ndk/build/cmake/android.toolchain.cmake new file mode 100644 index 000000000..e69de29bb diff --git a/llama-cpp-bindings-build/fixtures/android_ndk/toolchains/llvm/prebuilt/linux-x86_64/lib/clang/20/include/stddef.h b/llama-cpp-bindings-build/fixtures/android_ndk/toolchains/llvm/prebuilt/linux-x86_64/lib/clang/20/include/stddef.h new file mode 100644 index 000000000..e69de29bb diff --git a/llama-cpp-bindings-build/fixtures/bindings/llama_types.h b/llama-cpp-bindings-build/fixtures/bindings/llama_types.h new file mode 100644 index 000000000..2b2cd503c --- /dev/null +++ b/llama-cpp-bindings-build/fixtures/bindings/llama_types.h @@ -0,0 +1,15 @@ +#pragma once + +struct llama_context_params { + float defrag_thold; + unsigned int n_ctx; +}; + +struct llama_token_data { + int id; + float logit; +}; + +struct llama_sampler_chain_params { + int no_perf; +}; diff --git a/llama-cpp-bindings-build/fixtures/bindings/missing_privatized_field.h b/llama-cpp-bindings-build/fixtures/bindings/missing_privatized_field.h new file mode 100644 index 000000000..5429a180b --- /dev/null +++ b/llama-cpp-bindings-build/fixtures/bindings/missing_privatized_field.h @@ -0,0 +1,5 @@ +#pragma once + +struct llama_context_params { + unsigned int n_ctx; +}; diff --git a/llama-cpp-bindings-build/fixtures/bindings/wrapper.h b/llama-cpp-bindings-build/fixtures/bindings/wrapper.h new file mode 100644 index 000000000..8049a548d --- /dev/null +++ b/llama-cpp-bindings-build/fixtures/bindings/wrapper.h @@ -0,0 +1,8 @@ +#pragma once + +#include "llama_types.h" + +struct mtmd_context_params { + const char * image_marker; + const char * media_marker; +}; diff --git a/llama-cpp-bindings-build/fixtures/cmake_project/CMakeLists.txt b/llama-cpp-bindings-build/fixtures/cmake_project/CMakeLists.txt new file mode 100644 index 000000000..3ad4aca44 --- /dev/null +++ b/llama-cpp-bindings-build/fixtures/cmake_project/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 3.14) +project(llama_cpp_bindings_build_fixture C) + +if(NOT DEFINED FIXTURE_LIBRARY_NAME) + message(FATAL_ERROR "FIXTURE_LIBRARY_NAME must be defined") +endif() + +add_library(${FIXTURE_LIBRARY_NAME} STATIC fixture.c) + +install(TARGETS ${FIXTURE_LIBRARY_NAME} DESTINATION lib) diff --git a/llama-cpp-bindings-build/fixtures/cmake_project/fixture.c b/llama-cpp-bindings-build/fixtures/cmake_project/fixture.c new file mode 100644 index 000000000..abd6f0d07 --- /dev/null +++ b/llama-cpp-bindings-build/fixtures/cmake_project/fixture.c @@ -0,0 +1,3 @@ +int llama_cpp_bindings_build_fixture(void) { + return FIXTURE_VALUE; +} diff --git a/llama-cpp-bindings-build/fixtures/wrapper/vendor/vendored.h b/llama-cpp-bindings-build/fixtures/wrapper/vendor/vendored.h new file mode 100644 index 000000000..9e324d736 --- /dev/null +++ b/llama-cpp-bindings-build/fixtures/wrapper/vendor/vendored.h @@ -0,0 +1,3 @@ +#pragma once + +#define VENDORED_VALUE 7 diff --git a/llama-cpp-bindings-build/fixtures/wrapper/wrapper_fixture.cpp b/llama-cpp-bindings-build/fixtures/wrapper/wrapper_fixture.cpp new file mode 100644 index 000000000..ec19a8058 --- /dev/null +++ b/llama-cpp-bindings-build/fixtures/wrapper/wrapper_fixture.cpp @@ -0,0 +1,9 @@ +#include + +#ifndef WRAPPER_FIXTURE_DEFINE +#error "WRAPPER_FIXTURE_DEFINE must be defined" +#endif + +extern "C" auto llama_cpp_bindings_build_wrapper_fixture() -> int { + return VENDORED_VALUE; +} diff --git a/llama-cpp-bindings-build/src/android_architecture.rs b/llama-cpp-bindings-build/src/android_architecture.rs new file mode 100644 index 000000000..69c744e5b --- /dev/null +++ b/llama-cpp-bindings-build/src/android_architecture.rs @@ -0,0 +1,69 @@ +use crate::build_error::BuildError; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum AndroidArchitecture { + Aarch64, + Armv7, + X86_64, + X86, +} + +impl AndroidArchitecture { + /// # Errors + /// + /// Returns [`BuildError::UnsupportedAndroidTarget`] for an architecture the NDK does not + /// build for. + pub fn from_cargo_cfg(cargo_cfg_target_arch: &str) -> Result { + match cargo_cfg_target_arch { + "aarch64" => Ok(Self::Aarch64), + "arm" => Ok(Self::Armv7), + "x86_64" => Ok(Self::X86_64), + "x86" => Ok(Self::X86), + unsupported => Err(BuildError::UnsupportedAndroidTarget { + cargo_cfg_target_arch: unsupported.to_owned(), + }), + } + } + + #[must_use] + pub const fn abi(self) -> &'static str { + match self { + Self::Aarch64 => "arm64-v8a", + Self::Armv7 => "armeabi-v7a", + Self::X86_64 => "x86_64", + Self::X86 => "x86", + } + } + + #[must_use] + pub const fn ndk_prefix(self) -> &'static str { + match self { + Self::Aarch64 => "aarch64-linux-android", + Self::Armv7 => "arm-linux-androideabi", + Self::X86_64 => "x86_64-linux-android", + Self::X86 => "i686-linux-android", + } + } +} + +#[cfg(test)] +mod tests { + use super::AndroidArchitecture; + + #[test] + fn every_supported_android_architecture_maps_to_its_abi_and_ndk_prefix() { + assert_eq!( + ["aarch64", "arm", "x86_64", "x86"].map(|cargo_cfg_target_arch| { + AndroidArchitecture::from_cargo_cfg(cargo_cfg_target_arch) + .map(|architecture| [architecture.abi(), architecture.ndk_prefix()]) + .ok() + }), + [ + Some(["arm64-v8a", "aarch64-linux-android"]), + Some(["armeabi-v7a", "arm-linux-androideabi"]), + Some(["x86_64", "x86_64-linux-android"]), + Some(["x86", "i686-linux-android"]), + ] + ); + } +} diff --git a/llama-cpp-bindings-build/src/android_ndk.rs b/llama-cpp-bindings-build/src/android_ndk.rs index d6734fbc4..0621d438b 100644 --- a/llama-cpp-bindings-build/src/android_ndk.rs +++ b/llama-cpp-bindings-build/src/android_ndk.rs @@ -1,58 +1,96 @@ -use std::env; -use std::path::{Path, PathBuf}; - -use thiserror::Error; +use std::env::VarError; +use std::fs; +use std::fs::DirEntry; +use std::io; +use std::path::Path; +use std::path::PathBuf; + +use crate::android_architecture::AndroidArchitecture; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::host_os::HostOs; const DEFAULT_ANDROID_API_LEVEL: &str = "28"; -#[derive(Debug, Error)] -pub enum AndroidNdkDetectionError { - #[error("Android NDK not found for target {target_triple}. Set ANDROID_NDK_HOME.")] - NdkRootNotConfigured { - target_triple: String, - #[source] - source: env::VarError, - }, - #[error("Android NDK path does not exist: {path}")] - NdkRootMissing { path: PathBuf }, - #[error("Android NDK toolchain file not found: {path}")] - NdkToolchainFileMissing { path: PathBuf }, - #[error("Android NDK toolchain not found at: {path}")] - NdkToolchainDirectoryMissing { path: PathBuf }, - #[error("Unsupported host platform for Android NDK")] - UnsupportedHostPlatform, - #[error("Unsupported Android target architecture: {cargo_cfg_target_arch}")] - UnsupportedAndroidTarget { cargo_cfg_target_arch: String }, - #[error("ANDROID_PLATFORM is set but could not be read: {source}")] - AndroidPlatformUnreadable { - #[source] - source: env::VarError, - }, - #[error("Android NDK Clang directory could not be read at {path}: {source}")] - ClangDirectoryUnreadable { - path: PathBuf, - #[source] - source: std::io::Error, - }, - #[error("Android NDK Clang version directory was not found at {path}")] - ClangVersionDirectoryMissing { path: PathBuf }, - #[error("Android NDK Clang directory entry could not be read at {path}: {source}")] - ClangDirectoryEntryUnreadable { - path: PathBuf, - #[source] - source: std::io::Error, - }, - #[error("Android NDK Clang built-in include directory was not found at {path}")] - ClangBuiltinIncludesMissing { path: PathBuf }, +fn detect_ndk_path( + environment: &BuildEnvironment, + target_triple: &str, +) -> Result { + environment + .var("ANDROID_NDK_HOME") + .map_err(|source| BuildError::AndroidNdkRootNotConfigured { + target_triple: target_triple.to_owned(), + source, + }) +} + +fn validate_ndk_installation(ndk_path: &str) -> Result<(), BuildError> { + let ndk_path = Path::new(ndk_path); + + if !ndk_path.exists() { + return Err(BuildError::AndroidNdkRootMissing { + path: ndk_path.to_path_buf(), + }); + } + + let toolchain_file = ndk_path.join("build/cmake/android.toolchain.cmake"); + + if !toolchain_file.exists() { + return Err(BuildError::AndroidNdkToolchainFileMissing { + path: toolchain_file, + }); + } + + Ok(()) +} + +fn detect_api_level(environment: &BuildEnvironment) -> Result { + match environment.var("ANDROID_PLATFORM") { + Ok(platform) => Ok(platform + .strip_prefix("android-") + .unwrap_or(&platform) + .to_owned()), + Err(VarError::NotPresent) => Ok(DEFAULT_ANDROID_API_LEVEL.to_owned()), + Err(source) => Err(BuildError::AndroidPlatformUnreadable { source }), + } +} + +fn find_clang_builtin_includes(toolchain_path: &str) -> Result { + let clang_lib_path = PathBuf::from(toolchain_path).join("lib/clang"); + let version_dir = fs::read_dir(&clang_lib_path) + .and_then(Iterator::collect::, io::Error>>) + .map_err(|source| BuildError::AndroidNdkClangDirectoryUnreadable { + path: clang_lib_path.clone(), + source, + })? + .into_iter() + .map(|entry| entry.path()) + .find(|path| { + path.is_dir() + && path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| { + name.starts_with(|character: char| character.is_ascii_digit()) + }) + }) + .ok_or_else(|| BuildError::AndroidNdkClangVersionDirectoryMissing { + path: clang_lib_path.clone(), + })?; + let include_path = version_dir.join("include"); + + if !include_path.is_dir() { + return Err(BuildError::AndroidNdkClangBuiltinIncludesMissing { path: include_path }); + } + + Ok(include_path.to_string_lossy().into_owned()) } -#[derive(Debug)] +#[derive(Clone, Debug, Eq, PartialEq)] pub struct AndroidNdk { pub ndk_path: String, pub api_level: String, pub abi: &'static str, - pub host_tag: &'static str, - pub toolchain_path: String, pub sysroot: String, pub target_prefix: &'static str, pub clang_builtin_includes: String, @@ -61,362 +99,317 @@ pub struct AndroidNdk { impl AndroidNdk { /// # Errors /// - /// Returns [`AndroidNdkDetectionError`] when the NDK installation cannot be - /// located, an environment variable is missing or malformed, the target - /// architecture is unsupported, or the host platform is not supported by the NDK. + /// Returns the [`BuildError`] describing why the NDK installation cannot be located, an + /// environment variable is missing or malformed, the target architecture is unsupported, or + /// the host platform is not supported by the NDK. pub fn detect( + environment: &BuildEnvironment, target_triple: &str, cargo_cfg_target_arch: &str, - ) -> Result { - let ndk_path = detect_ndk_path(target_triple)?; + host_os: HostOs, + ) -> Result { + let ndk_path = detect_ndk_path(environment, target_triple)?; validate_ndk_installation(&ndk_path)?; let architecture = AndroidArchitecture::from_cargo_cfg(cargo_cfg_target_arch)?; - let api_level = detect_api_level()?; - let abi = architecture.abi(); - let host_tag = detect_host_tag()?; - let target_prefix = architecture.ndk_prefix(); + let api_level = detect_api_level(environment)?; + let host_tag = host_os.android_ndk_tag()?; let toolchain_path = format!("{ndk_path}/toolchains/llvm/prebuilt/{host_tag}"); if !Path::new(&toolchain_path).exists() { - return Err(AndroidNdkDetectionError::NdkToolchainDirectoryMissing { + return Err(BuildError::AndroidNdkToolchainDirectoryMissing { path: PathBuf::from(toolchain_path), }); } - let sysroot = format!("{toolchain_path}/sysroot"); let clang_builtin_includes = find_clang_builtin_includes(&toolchain_path)?; Ok(Self { ndk_path, api_level, - abi, - host_tag, - toolchain_path, - sysroot, - target_prefix, + abi: architecture.abi(), + sysroot: format!("{toolchain_path}/sysroot"), + target_prefix: architecture.ndk_prefix(), clang_builtin_includes, }) } + #[must_use] pub fn android_platform(&self) -> String { format!("android-{}", self.api_level) } + #[must_use] pub fn cmake_toolchain_file(&self) -> String { format!("{}/build/cmake/android.toolchain.cmake", self.ndk_path) } -} - -fn detect_ndk_path(target_triple: &str) -> Result { - env::var("ANDROID_NDK_HOME").map_err(|source| AndroidNdkDetectionError::NdkRootNotConfigured { - target_triple: target_triple.to_owned(), - source, - }) -} - -fn validate_ndk_installation(ndk_path: &str) -> Result<(), AndroidNdkDetectionError> { - let ndk_path = Path::new(ndk_path); - - if !ndk_path.exists() { - return Err(AndroidNdkDetectionError::NdkRootMissing { - path: ndk_path.to_path_buf(), - }); - } - - let toolchain_file = ndk_path.join("build/cmake/android.toolchain.cmake"); - - if !toolchain_file.exists() { - return Err(AndroidNdkDetectionError::NdkToolchainFileMissing { - path: toolchain_file, - }); - } - - Ok(()) -} -fn detect_api_level() -> Result { - match env::var("ANDROID_PLATFORM") { - Ok(platform) => Ok(platform - .strip_prefix("android-") - .unwrap_or(&platform) - .to_owned()), - Err(env::VarError::NotPresent) => Ok(DEFAULT_ANDROID_API_LEVEL.to_string()), - Err(source) => Err(AndroidNdkDetectionError::AndroidPlatformUnreadable { source }), + /// The clang arguments that parse headers the way the NDK compiles them for `target_triple`. + #[must_use] + pub fn clang_args(&self, target_triple: &str) -> Vec { + vec![ + format!("--sysroot={}", self.sysroot), + format!("-D__ANDROID_API__={}", self.api_level), + "-D__ANDROID__".to_owned(), + "-isystem".to_owned(), + self.clang_builtin_includes.clone(), + "-isystem".to_owned(), + format!("{}/usr/include/{}", self.sysroot, self.target_prefix), + "-isystem".to_owned(), + format!("{}/usr/include", self.sysroot), + "-include".to_owned(), + "stdbool.h".to_owned(), + "-include".to_owned(), + "stdint.h".to_owned(), + format!("--target={target_triple}"), + ] } } -const fn detect_host_tag() -> Result<&'static str, AndroidNdkDetectionError> { - if cfg!(target_os = "macos") { - Ok("darwin-x86_64") - } else if cfg!(target_os = "linux") { - Ok("linux-x86_64") - } else { - Err(AndroidNdkDetectionError::UnsupportedHostPlatform) - } -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum AndroidArchitecture { - Aarch64, - Armv7, - X86_64, - X86, -} - -impl AndroidArchitecture { - fn from_cargo_cfg(cargo_cfg_target_arch: &str) -> Result { - match cargo_cfg_target_arch { - "aarch64" => Ok(Self::Aarch64), - "arm" => Ok(Self::Armv7), - "x86_64" => Ok(Self::X86_64), - "x86" => Ok(Self::X86), - unsupported => Err(AndroidNdkDetectionError::UnsupportedAndroidTarget { - cargo_cfg_target_arch: unsupported.to_owned(), - }), - } - } - - const fn abi(self) -> &'static str { - match self { - Self::Aarch64 => "arm64-v8a", - Self::Armv7 => "armeabi-v7a", - Self::X86_64 => "x86_64", - Self::X86 => "x86", - } - } - - const fn ndk_prefix(self) -> &'static str { - match self { - Self::Aarch64 => "aarch64-linux-android", - Self::Armv7 => "arm-linux-androideabi", - Self::X86_64 => "x86_64-linux-android", - Self::X86 => "i686-linux-android", - } - } -} - -fn find_clang_builtin_includes(toolchain_path: &str) -> Result { - let clang_lib_path = PathBuf::from(toolchain_path).join("lib/clang"); - let entries = std::fs::read_dir(&clang_lib_path).map_err(|source| { - AndroidNdkDetectionError::ClangDirectoryUnreadable { - path: clang_lib_path.clone(), - source, - } - })?; - let mut version_dir = None; - for entry in entries { - let entry = - entry.map_err( - |source| AndroidNdkDetectionError::ClangDirectoryEntryUnreadable { - path: clang_lib_path.clone(), - source, - }, - )?; - let file_type = entry.file_type().map_err(|source| { - AndroidNdkDetectionError::ClangDirectoryEntryUnreadable { - path: entry.path(), - source, - } - })?; - if file_type.is_dir() - && entry - .file_name() - .to_str() - .is_some_and(|name| name.starts_with(|character: char| character.is_ascii_digit())) - { - version_dir = Some(entry); - break; - } - } - let version_dir = - version_dir.ok_or_else(|| AndroidNdkDetectionError::ClangVersionDirectoryMissing { - path: clang_lib_path.clone(), - })?; - - let include_path = clang_lib_path.join(version_dir.file_name()).join("include"); - - if !include_path.is_dir() { - return Err(AndroidNdkDetectionError::ClangBuiltinIncludesMissing { path: include_path }); - } - - Ok(include_path.to_string_lossy().into_owned()) -} - #[cfg(test)] -mod android_ndk_resolution_tests { +pub mod tests { + use std::ffi::OsString; + use std::os::unix::ffi::OsStringExt; + use std::path::Path; use std::path::PathBuf; - use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::atomic::AtomicUsize; + use std::sync::atomic::Ordering; + + use enumflags2::BitFlags; - use super::AndroidArchitecture; use super::AndroidNdk; - use super::AndroidNdkDetectionError; use super::find_clang_builtin_includes; use super::validate_ndk_installation; + use crate::build_environment::BuildEnvironment; + use crate::build_environment::tests::environment_of; + use crate::host_os::HostOs; + + pub const FIXTURE_ANDROID_NDK: &str = + concat!(env!("CARGO_MANIFEST_DIR"), "/fixtures/android_ndk"); + + const FIXTURE_TOOLCHAIN: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/fixtures/android_ndk/toolchains/llvm/prebuilt/linux-x86_64" + ); static NEXT_DIRECTORY_ID: AtomicUsize = AtomicUsize::new(0); fn temporary_directory(name: &str) -> PathBuf { let id = NEXT_DIRECTORY_ID.fetch_add(1, Ordering::Relaxed); + std::env::temp_dir().join(format!( "llama-cpp-bindings-{name}-{}-{id}", std::process::id() )) } - #[test] - fn every_supported_android_architecture_maps_to_its_abi_and_ndk_prefix() { - let architectures = [ - ("aarch64", "arm64-v8a", "aarch64-linux-android"), - ("arm", "armeabi-v7a", "arm-linux-androideabi"), - ("x86_64", "x86_64", "x86_64-linux-android"), - ("x86", "x86", "i686-linux-android"), - ]; - - for (cargo_cfg_target_arch, abi, prefix) in architectures { - let architecture = AndroidArchitecture::from_cargo_cfg(cargo_cfg_target_arch) - .expect("the architecture is supported"); - - assert_eq!(architecture.abi(), abi); - assert_eq!(architecture.ndk_prefix(), prefix); - } + fn ndk_at(root: &Path) -> BuildEnvironment { + environment_of( + BitFlags::empty(), + [("ANDROID_NDK_HOME", OsString::from(root))], + ) } - #[test] - fn an_unsupported_android_architecture_preserves_what_cargo_reported() { - let error = AndroidArchitecture::from_cargo_cfg("riscv64") - .expect_err("riscv64 has no NDK toolchain in this build"); - - assert!(matches!( - error, - AndroidNdkDetectionError::UnsupportedAndroidTarget { cargo_cfg_target_arch } - if cargo_cfg_target_arch == "riscv64" - )); + fn detection_failure( + environment: &BuildEnvironment, + cargo_cfg_target_arch: &str, + host_os: HostOs, + ) -> Option { + AndroidNdk::detect( + environment, + "aarch64-linux-android", + cargo_cfg_target_arch, + host_os, + ) + .err() + .map(|error| error.to_string()) } #[test] - fn android_ndk_paths_derive_from_the_resolved_root_and_api_level() { - let ndk = AndroidNdk { - ndk_path: "/opt/android-ndk".to_owned(), - api_level: "35".to_owned(), - abi: "arm64-v8a", - host_tag: "linux-x86_64", - toolchain_path: "/opt/android-ndk/toolchains/llvm/prebuilt/linux-x86_64".to_owned(), - sysroot: "/opt/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/sysroot".to_owned(), - target_prefix: "aarch64-linux-android", - clang_builtin_includes: "/opt/android-ndk/lib/clang/20/include".to_owned(), - }; + fn an_installed_ndk_is_detected_with_the_platform_api_level() { + let detected = AndroidNdk::detect( + &environment_of( + BitFlags::empty(), + [ + ("ANDROID_NDK_HOME", OsString::from(FIXTURE_ANDROID_NDK)), + ("ANDROID_PLATFORM", OsString::from("android-30")), + ], + ), + "aarch64-linux-android", + "aarch64", + HostOs::Linux, + ) + .unwrap(); - assert_eq!(ndk.android_platform(), "android-35"); assert_eq!( - ndk.cmake_toolchain_file(), - "/opt/android-ndk/build/cmake/android.toolchain.cmake" + [detected.android_platform(), detected.cmake_toolchain_file(),], + [ + "android-30".to_owned(), + format!("{FIXTURE_ANDROID_NDK}/build/cmake/android.toolchain.cmake"), + ] + ); + assert_eq!( + detected, + AndroidNdk { + ndk_path: FIXTURE_ANDROID_NDK.to_owned(), + api_level: "30".to_owned(), + abi: "arm64-v8a", + sysroot: format!("{FIXTURE_TOOLCHAIN}/sysroot"), + target_prefix: "aarch64-linux-android", + clang_builtin_includes: format!("{FIXTURE_TOOLCHAIN}/lib/clang/20/include"), + } ); } #[test] - fn ndk_validation_rejects_a_missing_root_directory() { - let root = temporary_directory("missing-ndk-root"); - - let error = validate_ndk_installation(root.to_str().expect("UTF-8 temporary path")) - .expect_err("missing NDK root must fail"); + fn the_api_level_is_read_with_or_without_the_android_prefix_and_defaults_to_28() { + let api_level_of = |environment: &BuildEnvironment| { + AndroidNdk::detect( + environment, + "aarch64-linux-android", + "aarch64", + HostOs::Linux, + ) + .map(|ndk| ndk.api_level) + .ok() + }; - assert!(matches!( - error, - AndroidNdkDetectionError::NdkRootMissing { path } if path == root - )); + assert_eq!( + [ + api_level_of(&environment_of( + BitFlags::empty(), + [ + ("ANDROID_NDK_HOME", OsString::from(FIXTURE_ANDROID_NDK)), + ("ANDROID_PLATFORM", OsString::from("33")), + ] + )), + api_level_of(&ndk_at(Path::new(FIXTURE_ANDROID_NDK))), + ], + [Some("33".to_owned()), Some("28".to_owned())] + ); } #[test] - fn ndk_validation_rejects_a_root_without_a_cmake_toolchain() { - let root = temporary_directory("ndk-without-toolchain"); - std::fs::create_dir_all(&root).expect("temporary NDK root must be created"); + fn detection_reports_every_way_the_ndk_cannot_be_used() { + let fixture_ndk = Path::new(FIXTURE_ANDROID_NDK); + let missing_root = temporary_directory("missing-ndk-root"); - let error = validate_ndk_installation(root.to_str().expect("UTF-8 temporary path")) - .expect_err("missing toolchain file must fail"); - - assert!(matches!( - error, - AndroidNdkDetectionError::NdkToolchainFileMissing { path } - if path == root.join("build/cmake/android.toolchain.cmake") - )); - std::fs::remove_dir_all(root).expect("temporary NDK root must be removed"); + assert_eq!( + [ + detection_failure( + &environment_of(BitFlags::empty(), []), + "aarch64", + HostOs::Linux + ), + detection_failure(&ndk_at(&missing_root), "aarch64", HostOs::Linux), + detection_failure(&ndk_at(fixture_ndk), "riscv64", HostOs::Linux), + detection_failure( + &environment_of( + BitFlags::empty(), + [ + ("ANDROID_NDK_HOME", OsString::from(FIXTURE_ANDROID_NDK)), + ("ANDROID_PLATFORM", OsString::from_vec(vec![0xff])), + ] + ), + "aarch64", + HostOs::Linux + ), + detection_failure(&ndk_at(fixture_ndk), "aarch64", HostOs::Unsupported), + detection_failure(&ndk_at(fixture_ndk), "aarch64", HostOs::MacOs), + ], + [ + Some( + "Android NDK not found for target aarch64-linux-android. Set ANDROID_NDK_HOME." + .to_owned() + ), + Some(format!( + "Android NDK path does not exist: {}", + missing_root.display() + )), + Some("Unsupported Android target architecture: riscv64".to_owned()), + Some( + "ANDROID_PLATFORM is set but could not be read: environment variable was not \ + valid unicode: \"\\xFF\"" + .to_owned() + ), + Some("Unsupported host platform for Android NDK".to_owned()), + Some(format!( + "Android NDK toolchain not found at: \ + {FIXTURE_ANDROID_NDK}/toolchains/llvm/prebuilt/darwin-x86_64" + )), + ] + ); } #[test] - fn ndk_validation_accepts_a_root_with_a_cmake_toolchain() { - let root = temporary_directory("valid-ndk-root"); - let toolchain = root.join("build/cmake/android.toolchain.cmake"); - std::fs::create_dir_all(toolchain.parent().expect("toolchain parent")) - .expect("toolchain directory must be created"); - std::fs::write(&toolchain, "").expect("toolchain file must be created"); - - validate_ndk_installation(root.to_str().expect("UTF-8 temporary path")) - .expect("valid NDK root must pass"); + fn ndk_validation_rejects_a_root_without_a_cmake_toolchain() { + let root = temporary_directory("ndk-without-toolchain"); + std::fs::create_dir_all(&root).unwrap(); - std::fs::remove_dir_all(root).expect("temporary NDK root must be removed"); + assert_eq!( + validate_ndk_installation(root.to_str().unwrap()) + .err() + .map(|error| error.to_string()), + Some(format!( + "Android NDK toolchain file not found: {}", + root.join("build/cmake/android.toolchain.cmake").display() + )) + ); + std::fs::remove_dir_all(root).unwrap(); } #[test] - fn clang_builtin_include_resolution_rejects_a_missing_clang_directory() { - let toolchain = temporary_directory("missing-clang-directory"); + fn detection_rejects_a_toolchain_without_a_clang_directory() { + let root = temporary_directory("ndk-without-clang"); + let toolchain = root.join("toolchains/llvm/prebuilt/linux-x86_64"); + std::fs::create_dir_all(root.join("build/cmake")).unwrap(); + std::fs::write(root.join("build/cmake/android.toolchain.cmake"), "").unwrap(); + std::fs::create_dir_all(&toolchain).unwrap(); - let error = find_clang_builtin_includes(toolchain.to_str().expect("UTF-8 temporary path")) - .expect_err("missing Clang directory must fail"); - - assert!(matches!( - error, - AndroidNdkDetectionError::ClangDirectoryUnreadable { path, .. } - if path == toolchain.join("lib/clang") - )); + assert_eq!( + detection_failure(&ndk_at(&root), "aarch64", HostOs::Linux), + Some(format!( + "Android NDK Clang directory could not be listed at {}: No such file or directory \ + (os error 2)", + toolchain.join("lib/clang").display() + )) + ); + std::fs::remove_dir_all(root).unwrap(); } #[test] fn clang_builtin_include_resolution_requires_a_version_directory() { let toolchain = temporary_directory("clang-without-version"); - std::fs::create_dir_all(toolchain.join("lib/clang/not-a-version")) - .expect("Clang directory must be created"); - - let error = find_clang_builtin_includes(toolchain.to_str().expect("UTF-8 temporary path")) - .expect_err("missing version directory must fail"); - - assert!(matches!( - error, - AndroidNdkDetectionError::ClangVersionDirectoryMissing { path } - if path == toolchain.join("lib/clang") - )); - std::fs::remove_dir_all(toolchain).expect("temporary toolchain must be removed"); + std::fs::create_dir_all(toolchain.join("lib/clang/not-a-version")).unwrap(); + std::fs::write(toolchain.join("lib/clang/21"), "").unwrap(); + + assert_eq!( + find_clang_builtin_includes(toolchain.to_str().unwrap()) + .err() + .map(|error| error.to_string()), + Some(format!( + "Android NDK Clang version directory was not found at {}", + toolchain.join("lib/clang").display() + )) + ); + std::fs::remove_dir_all(toolchain).unwrap(); } #[test] fn clang_builtin_include_resolution_requires_an_include_directory() { let toolchain = temporary_directory("clang-without-includes"); - std::fs::create_dir_all(toolchain.join("lib/clang/20")) - .expect("Clang version directory must be created"); - - let error = find_clang_builtin_includes(toolchain.to_str().expect("UTF-8 temporary path")) - .expect_err("missing built-in includes must fail"); - - assert!(matches!( - error, - AndroidNdkDetectionError::ClangBuiltinIncludesMissing { path } - if path == toolchain.join("lib/clang/20/include") - )); - std::fs::remove_dir_all(toolchain).expect("temporary toolchain must be removed"); - } - - #[test] - fn clang_builtin_include_resolution_returns_the_version_include_directory() { - let toolchain = temporary_directory("clang-with-includes"); - let include = toolchain.join("lib/clang/20/include"); - std::fs::create_dir_all(&include).expect("Clang include directory must be created"); - - let resolved = - find_clang_builtin_includes(toolchain.to_str().expect("UTF-8 temporary path")) - .expect("built-in includes must resolve"); + std::fs::create_dir_all(toolchain.join("lib/clang/20")).unwrap(); - assert_eq!(PathBuf::from(resolved), include); - std::fs::remove_dir_all(toolchain).expect("temporary toolchain must be removed"); + assert_eq!( + find_clang_builtin_includes(toolchain.to_str().unwrap()) + .err() + .map(|error| error.to_string()), + Some(format!( + "Android NDK Clang built-in include directory was not found at {}", + toolchain.join("lib/clang/20/include").display() + )) + ); + std::fs::remove_dir_all(toolchain).unwrap(); } } diff --git a/llama-cpp-bindings-build/src/apple_variant.rs b/llama-cpp-bindings-build/src/apple_variant.rs deleted file mode 100644 index 2996a38fb..000000000 --- a/llama-cpp-bindings-build/src/apple_variant.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub enum AppleVariant { - MacOS, - Other, -} diff --git a/llama-cpp-bindings-build/src/bindgen_config.rs b/llama-cpp-bindings-build/src/bindgen_config.rs deleted file mode 100644 index 371e9f2fd..000000000 --- a/llama-cpp-bindings-build/src/bindgen_config.rs +++ /dev/null @@ -1,271 +0,0 @@ -use std::path::Path; -use std::sync::Arc; -use std::sync::atomic::AtomicBool; -use std::sync::atomic::Ordering; - -use llama_cpp_wrapper_sources::fault_injection_define::FAULT_INJECTION_DEFINE; - -use crate::BuildError; -use crate::android_ndk::AndroidNdk; -use crate::debug_log; -use crate::target_os::TargetOs; - -const DEPRECATED_FUNCTIONS: &[&str] = &[ - "ggml_add1", - "ggml_add1_inplace", - "ggml_flash_attn_ext_set_prec", - "ggml_mul_mat_set_prec", - "ggml_rope_custom", - "ggml_rope_custom_inplace", - "ggml_type_sizef", - "ggml_upscale_ext", - "llama_add_bos_token", - "llama_add_eos_token", - "llama_copy_state_data", - "llama_free_model", - "llama_get_state_size", - "llama_load_model_from_file", - "llama_load_session_file", - "llama_n_ctx_train", - "llama_n_embd", - "llama_n_head", - "llama_n_layer", - "llama_n_vocab", - "llama_new_context_with_model", - "llama_sampler_init_grammar_lazy", - "llama_save_session_file", - "llama_set_state_data", - "llama_set_warmup", - "llama_token_bos", - "llama_token_cls", - "llama_token_eos", - "llama_token_eot", - "llama_token_fim_mid", - "llama_token_fim_pad", - "llama_token_fim_pre", - "llama_token_fim_rep", - "llama_token_fim_sep", - "llama_token_fim_suf", - "llama_token_get_attr", - "llama_token_get_score", - "llama_token_get_text", - "llama_token_is_control", - "llama_token_is_eog", - "llama_token_nl", - "llama_token_pad", - "llama_token_sep", - "llama_vocab_cls", - "mtmd_encode", - "mtmd_helper_model_can_chat", - "mtmd_helper_support_video", - "mtmd_image_tokens_get_nx", - "mtmd_image_tokens_get_ny", -]; - -const COMPARED_STRUCTS: &[&str] = &["llama_logit_bias", "llama_token_data"]; - -struct PrivatizedField { - type_name: &'static str, - field_name: &'static str, -} - -const PRIVATIZED_FIELDS: &[PrivatizedField] = &[ - PrivatizedField { - type_name: "llama_context_params", - field_name: "defrag_thold", - }, - PrivatizedField { - type_name: "mtmd_context_params", - field_name: "image_marker", - }, -]; - -fn derives_for(type_name: &str) -> Vec { - if COMPARED_STRUCTS.contains(&type_name) { - vec!["PartialEq".to_owned()] - } else { - Vec::new() - } -} - -#[derive(Clone, Debug)] -struct BindingCallbacks { - privatized_field_hits: Arc>, -} - -impl BindingCallbacks { - fn new() -> Self { - Self { - privatized_field_hits: Arc::new( - PRIVATIZED_FIELDS - .iter() - .map(|_| AtomicBool::new(false)) - .collect(), - ), - } - } - - fn verify_every_privatized_field_was_found(&self) -> Result<(), BuildError> { - for (field, was_found) in PRIVATIZED_FIELDS - .iter() - .zip(self.privatized_field_hits.iter()) - { - if !was_found.load(Ordering::Relaxed) { - return Err(BuildError::PrivatizedFieldMissing { - type_name: field.type_name, - field_name: field.field_name, - }); - } - } - - Ok(()) - } -} - -impl bindgen::callbacks::ParseCallbacks for BindingCallbacks { - fn header_file(&self, filename: &str) { - println!("cargo:rerun-if-changed={filename}"); - } - - fn include_file(&self, filename: &str) { - println!("cargo:rerun-if-changed={filename}"); - } - - fn read_env_var(&self, key: &str) { - println!("cargo:rerun-if-env-changed={key}"); - } - - fn add_derives(&self, info: &bindgen::callbacks::DeriveInfo<'_>) -> Vec { - derives_for(info.name) - } - - fn field_visibility( - &self, - info: bindgen::callbacks::FieldInfo<'_>, - ) -> Option { - for (field, was_found) in PRIVATIZED_FIELDS - .iter() - .zip(self.privatized_field_hits.iter()) - { - if field.type_name == info.type_name && field.field_name == info.field_name { - was_found.store(true, Ordering::Relaxed); - - return Some(bindgen::FieldVisibilityKind::Private); - } - } - - None - } -} - -pub fn generate_bindings( - llama_src: &Path, - out_dir: &Path, - target_os: TargetOs, - target_triple: &str, - android_ndk: Option<&AndroidNdk>, -) -> Result<(), BuildError> { - let callbacks = BindingCallbacks::new(); - let mut builder = create_base_builder(llama_src, callbacks.clone()); - - if target_os.is_android() - && let Some(ndk) = android_ndk - { - builder = configure_android_bindgen(builder, ndk, target_triple); - } - - let bindings = builder.generate().map_err(BuildError::Bindgen)?; - - callbacks.verify_every_privatized_field_was_found()?; - - bindings - .write_to_file(out_dir.join("bindings.rs")) - .map_err(BuildError::BindingsWrite)?; - - debug_log!("Bindings Created"); - - Ok(()) -} - -fn create_base_builder(llama_src: &Path, callbacks: BindingCallbacks) -> bindgen::Builder { - let mut builder = bindgen::Builder::default() - .header("wrapper.h") - .clang_arg(format!("-I{}", llama_src.display())) - .clang_arg(format!("-I{}", llama_src.join("include").display())) - .clang_arg(format!("-I{}", llama_src.join("ggml/include").display())) - .parse_callbacks(Box::new(callbacks)) - .generate_comments(false) - .allowlist_function("ggml_.*") - .allowlist_type("ggml_.*") - .allowlist_var("GGML_MAX_DIMS") - .allowlist_function("gguf_.*") - .allowlist_type("gguf_.*") - .allowlist_function("llama_.*") - .allowlist_type("llama_.*") - .allowlist_var("LLAMA_TOKEN_NULL") - .allowlist_var("LLAMA_DRY_MAX_SEQ_BREAKERS") - .allowlist_function("mtmd_.*") - .allowlist_type("mtmd_.*") - .blocklist_function("ggml_fopen") - .blocklist_function("llama_adapter_lora_init_from_file_ptr") - .blocklist_function("gguf_init_from_file_ptr") - .blocklist_function("gguf_write_to_file_ptr") - .blocklist_function("llama_model_load_from_file_ptr") - .blocklist_type("FILE") - .blocklist_type("_IO_.*") - .prepend_enum_name(false); - - for function in DEPRECATED_FUNCTIONS { - builder = builder.blocklist_function(function); - } - - if cfg!(feature = "fault-injection") { - builder = builder.clang_arg(format!("-D{FAULT_INJECTION_DEFINE}")); - } - - builder -} - -fn configure_android_bindgen( - mut builder: bindgen::Builder, - ndk: &AndroidNdk, - target_triple: &str, -) -> bindgen::Builder { - builder = builder - .clang_arg(format!("--sysroot={}", ndk.sysroot)) - .clang_arg(format!("-D__ANDROID_API__={}", ndk.api_level)) - .clang_arg("-D__ANDROID__"); - - builder = builder - .clang_arg("-isystem") - .clang_arg(&ndk.clang_builtin_includes); - - builder = builder - .clang_arg("-isystem") - .clang_arg(format!("{}/usr/include/{}", ndk.sysroot, ndk.target_prefix)) - .clang_arg("-isystem") - .clang_arg(format!("{}/usr/include", ndk.sysroot)) - .clang_arg("-include") - .clang_arg("stdbool.h") - .clang_arg("-include") - .clang_arg("stdint.h"); - - builder.clang_arg(format!("--target={target_triple}")) -} - -#[cfg(test)] -mod derives_for_tests { - use super::derives_for; - - #[test] - fn only_the_structs_the_bindings_compare_derive_partial_eq() { - assert_eq!( - ["llama_token_data", "llama_logit_bias", "llama_sampler_i"].map(derives_for), - [ - vec!["PartialEq".to_owned()], - vec!["PartialEq".to_owned()], - Vec::new() - ] - ); - } -} diff --git a/llama-cpp-bindings-build/src/binding_callbacks.rs b/llama-cpp-bindings-build/src/binding_callbacks.rs new file mode 100644 index 000000000..38b967646 --- /dev/null +++ b/llama-cpp-bindings-build/src/binding_callbacks.rs @@ -0,0 +1,123 @@ +use std::cell::Cell; +use std::cell::RefCell; +use std::path::PathBuf; +use std::rc::Rc; + +use bindgen::FieldVisibilityKind; +use bindgen::callbacks::DeriveInfo; +use bindgen::callbacks::FieldInfo; +use bindgen::callbacks::ParseCallbacks; + +use crate::build_error::BuildError; +use crate::cargo_directive::CargoDirective; +use crate::privatized_field::PrivatizedField; + +const COMPARED_STRUCTS: &[&str] = &["llama_logit_bias", "llama_token_data"]; + +fn derives_for(type_name: &str) -> Vec { + if COMPARED_STRUCTS.contains(&type_name) { + vec!["PartialEq".to_owned()] + } else { + Vec::new() + } +} + +/// Observes bindgen while it parses: keeps the [`PrivatizedField::ALL`] fields private, derives +/// `PartialEq` for the structs the bindings compare, and records what the bindings depend on. +#[derive(Clone, Debug)] +pub struct BindingCallbacks { + privatized_field_hits: Rc<[Cell]>, + rebuild_triggers: Rc>>, +} + +impl Default for BindingCallbacks { + fn default() -> Self { + Self { + privatized_field_hits: PrivatizedField::ALL + .iter() + .map(|_| Cell::new(false)) + .collect(), + rebuild_triggers: Rc::default(), + } + } +} + +impl BindingCallbacks { + /// # Errors + /// + /// Returns [`BuildError::PrivatizedFieldMissing`] for the first privatized field bindgen + /// never saw. + pub fn verify_every_privatized_field_was_found(&self) -> Result<(), BuildError> { + PrivatizedField::ALL + .iter() + .zip(self.privatized_field_hits.iter()) + .find(|(_, was_found)| !was_found.get()) + .map_or(Ok(()), |(field, _)| { + Err(BuildError::PrivatizedFieldMissing { + type_name: field.type_name, + field_name: field.field_name, + }) + }) + } + + /// The headers and environment variables bindgen read, as cargo rebuild triggers. + #[must_use] + pub fn take_rebuild_triggers(&self) -> Vec { + self.rebuild_triggers.take() + } +} + +impl ParseCallbacks for BindingCallbacks { + fn header_file(&self, filename: &str) { + self.rebuild_triggers + .borrow_mut() + .push(CargoDirective::RerunIfChanged(PathBuf::from(filename))); + } + + fn include_file(&self, filename: &str) { + self.rebuild_triggers + .borrow_mut() + .push(CargoDirective::RerunIfChanged(PathBuf::from(filename))); + } + + fn read_env_var(&self, key: &str) { + self.rebuild_triggers + .borrow_mut() + .push(CargoDirective::RerunIfEnvChanged(key.to_owned())); + } + + fn add_derives(&self, info: &DeriveInfo<'_>) -> Vec { + derives_for(info.name) + } + + fn field_visibility(&self, info: FieldInfo<'_>) -> Option { + PrivatizedField::ALL + .iter() + .zip(self.privatized_field_hits.iter()) + .find(|(field, _)| { + field.type_name == info.type_name && field.field_name == info.field_name + }) + .map(|(_, was_found)| { + was_found.set(true); + + FieldVisibilityKind::Private + }) + } +} + +#[cfg(test)] +mod tests { + use super::derives_for; + + #[test] + fn only_the_structs_the_bindings_compare_derive_partial_eq() { + assert_eq!( + ["llama_token_data", "llama_logit_bias", "llama_sampler_i"].map(derives_for), + [ + vec!["PartialEq".to_owned()], + vec!["PartialEq".to_owned()], + Vec::new() + ] + ); + } +} diff --git a/llama-cpp-bindings-build/src/bindings_generation.rs b/llama-cpp-bindings-build/src/bindings_generation.rs new file mode 100644 index 000000000..93003ed48 --- /dev/null +++ b/llama-cpp-bindings-build/src/bindings_generation.rs @@ -0,0 +1,295 @@ +use std::path::PathBuf; + +use bindgen::Builder; +use llama_cpp_wrapper_sources::fault_injection_define::FAULT_INJECTION_DEFINE; + +use crate::binding_callbacks::BindingCallbacks; +use crate::build_context::BuildContext; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::build_feature::BuildFeature; +use crate::cargo_directive::CargoDirective; +use crate::target_os::TargetOs; + +const DEPRECATED_FUNCTIONS: &[&str] = &[ + "ggml_add1", + "ggml_add1_inplace", + "ggml_flash_attn_ext_set_prec", + "ggml_mul_mat_set_prec", + "ggml_rope_custom", + "ggml_rope_custom_inplace", + "ggml_type_sizef", + "ggml_upscale_ext", + "llama_add_bos_token", + "llama_add_eos_token", + "llama_copy_state_data", + "llama_free_model", + "llama_get_state_size", + "llama_load_model_from_file", + "llama_load_session_file", + "llama_n_ctx_train", + "llama_n_embd", + "llama_n_head", + "llama_n_layer", + "llama_n_vocab", + "llama_new_context_with_model", + "llama_sampler_init_grammar_lazy", + "llama_save_session_file", + "llama_set_state_data", + "llama_set_warmup", + "llama_token_bos", + "llama_token_cls", + "llama_token_eos", + "llama_token_eot", + "llama_token_fim_mid", + "llama_token_fim_pad", + "llama_token_fim_pre", + "llama_token_fim_rep", + "llama_token_fim_sep", + "llama_token_fim_suf", + "llama_token_get_attr", + "llama_token_get_score", + "llama_token_get_text", + "llama_token_is_control", + "llama_token_is_eog", + "llama_token_nl", + "llama_token_pad", + "llama_token_sep", + "llama_vocab_cls", + "mtmd_encode", + "mtmd_helper_model_can_chat", + "mtmd_helper_support_video", + "mtmd_image_tokens_get_nx", + "mtmd_image_tokens_get_ny", +]; + +fn configured_builder() -> Builder { + let builder = Builder::default() + .generate_comments(false) + .allowlist_function("ggml_.*") + .allowlist_type("ggml_.*") + .allowlist_var("GGML_MAX_DIMS") + .allowlist_function("gguf_.*") + .allowlist_type("gguf_.*") + .allowlist_function("llama_.*") + .allowlist_type("llama_.*") + .allowlist_var("LLAMA_TOKEN_NULL") + .allowlist_var("LLAMA_DRY_MAX_SEQ_BREAKERS") + .allowlist_function("mtmd_.*") + .allowlist_type("mtmd_.*") + .blocklist_function("ggml_fopen") + .blocklist_function("llama_adapter_lora_init_from_file_ptr") + .blocklist_function("gguf_init_from_file_ptr") + .blocklist_function("gguf_write_to_file_ptr") + .blocklist_function("llama_model_load_from_file_ptr") + .blocklist_type("FILE") + .blocklist_type("_IO_.*") + .prepend_enum_name(false); + + DEPRECATED_FUNCTIONS + .iter() + .fold(builder, |builder, function| { + builder.blocklist_function(function) + }) +} + +fn fault_injection_args(environment: &BuildEnvironment) -> Vec { + if environment.has(BuildFeature::FaultInjection) { + vec![format!("-D{FAULT_INJECTION_DEFINE}")] + } else { + Vec::new() + } +} + +fn target_args(context: &BuildContext) -> Vec { + match &context.target_os { + TargetOs::Android(android_ndk) => android_ndk.clang_args(&context.target_triple), + TargetOs::Apple | TargetOs::Linux => Vec::new(), + } +} + +/// Generates the Rust bindings of `header` with bindgen into `output`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct BindingsGeneration { + pub header: PathBuf, + pub clang_args: Vec, + pub output: PathBuf, +} + +impl BindingsGeneration { + #[must_use] + pub fn of(context: &BuildContext, environment: &BuildEnvironment) -> Self { + let llama_src = context.llama_src(); + + Self { + header: context.sys_dir.join("wrapper.h"), + clang_args: [ + vec![ + format!("-I{}", llama_src.display()), + format!("-I{}", llama_src.join("include").display()), + format!("-I{}", llama_src.join("ggml/include").display()), + ], + fault_injection_args(environment), + target_args(context), + ] + .concat(), + output: context.out_dir.join("bindings.rs"), + } + } + + /// Writes the bindings and returns the cargo directives that rebuild them when what they + /// were generated from changes. + /// + /// # Errors + /// + /// Returns [`BuildError::Bindgen`] when bindgen cannot parse the header, + /// [`BuildError::PrivatizedFieldMissing`] when a field the bindings keep private no longer + /// exists, or [`BuildError::BindingsWrite`] when `output` cannot be written. + pub fn generate(&self) -> Result, BuildError> { + let callbacks = BindingCallbacks::default(); + let bindings = configured_builder() + .header(self.header.display().to_string()) + .clang_args(&self.clang_args) + .parse_callbacks(Box::new(callbacks.clone())) + .generate() + .map_err(BuildError::Bindgen)?; + + callbacks.verify_every_privatized_field_was_found()?; + bindings + .write_to_file(&self.output) + .map_err(BuildError::BindingsWrite)?; + + Ok([ + callbacks.take_rebuild_triggers(), + vec![CargoDirective::Debug("Bindings Created".to_owned())], + ] + .concat()) + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use enumflags2::BitFlags; + use llama_cpp_wrapper_sources::fault_injection_define::FAULT_INJECTION_DEFINE; + + use super::BindingsGeneration; + use crate::android_ndk::tests::FIXTURE_ANDROID_NDK; + use crate::build_context::tests::ANDROID_OVERRIDES; + use crate::build_context::tests::cargo_environment; + use crate::build_context::tests::context_of; + use crate::build_feature::BuildFeature; + + const FIXTURE_BINDINGS: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/fixtures/bindings"); + + fn planned(features: BitFlags, overrides: &[(&str, &str)]) -> BindingsGeneration { + let environment = cargo_environment(features, overrides); + + BindingsGeneration::of(&context_of(&environment), &environment) + } + + fn llama_cpp_includes() -> Vec { + vec![ + "-I/sys/llama.cpp".to_owned(), + "-I/sys/llama.cpp/include".to_owned(), + "-I/sys/llama.cpp/ggml/include".to_owned(), + ] + } + + fn generation_failure(header: &str, output: PathBuf) -> Option { + BindingsGeneration { + header: PathBuf::from(FIXTURE_BINDINGS).join(header), + clang_args: Vec::new(), + output, + } + .generate() + .err() + .map(|error| error.to_string()) + } + + #[test] + fn the_wrapper_header_is_parsed_against_the_llama_cpp_headers() { + assert_eq!( + planned(BitFlags::empty(), &[]), + BindingsGeneration { + header: PathBuf::from("/sys/wrapper.h"), + clang_args: llama_cpp_includes(), + output: PathBuf::from("/out/bindings.rs"), + } + ); + } + + #[test] + fn fault_injection_and_android_targets_add_their_clang_arguments() { + let toolchain = format!("{FIXTURE_ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64"); + + assert_eq!( + planned( + BitFlags::from(BuildFeature::FaultInjection), + &ANDROID_OVERRIDES + ) + .clang_args, + [ + llama_cpp_includes(), + vec![ + format!("-D{FAULT_INJECTION_DEFINE}"), + format!("--sysroot={toolchain}/sysroot"), + "-D__ANDROID_API__=28".to_owned(), + "-D__ANDROID__".to_owned(), + "-isystem".to_owned(), + format!("{toolchain}/lib/clang/20/include"), + "-isystem".to_owned(), + format!("{toolchain}/sysroot/usr/include/aarch64-linux-android"), + "-isystem".to_owned(), + format!("{toolchain}/sysroot/usr/include"), + "-include".to_owned(), + "stdbool.h".to_owned(), + "-include".to_owned(), + "stdint.h".to_owned(), + "--target=aarch64-linux-android".to_owned(), + ], + ] + .concat() + ); + } + + #[test] + fn a_missing_header_is_reported_by_bindgen() { + assert_eq!( + generation_failure("missing.h", PathBuf::from("/out/bindings.rs")), + Some(format!( + "bindgen failed: header '{FIXTURE_BINDINGS}/missing.h' does not exist." + )) + ); + } + + #[test] + fn a_header_without_a_privatized_field_is_refused() { + assert_eq!( + generation_failure( + "missing_privatized_field.h", + PathBuf::from("/out/bindings.rs") + ), + Some( + "bindgen never saw field llama_context_params::defrag_thold, so it can no longer \ + be kept private" + .to_owned() + ) + ); + } + + #[test] + fn bindings_that_cannot_be_written_are_reported() { + assert_eq!( + generation_failure( + "wrapper.h", + PathBuf::from(FIXTURE_BINDINGS).join("missing-directory/bindings.rs") + ), + Some( + "generated bindings could not be written: No such file or directory (os error 2)" + .to_owned() + ) + ); + } +} diff --git a/llama-cpp-bindings-build/src/build_context.rs b/llama-cpp-bindings-build/src/build_context.rs new file mode 100644 index 000000000..e251c4322 --- /dev/null +++ b/llama-cpp-bindings-build/src/build_context.rs @@ -0,0 +1,255 @@ +use std::path::PathBuf; + +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::build_feature::BuildFeature; +use crate::cargo_directive::CargoDirective; +use crate::host_os::HostOs; +use crate::target_os::TargetOs; + +fn native_profile(cargo_profile: &str) -> String { + match cargo_profile { + "debug" | "release" => "Release".to_owned(), + other => other.to_owned(), + } +} + +/// Where and for what the native libraries are built, as cargo describes the build script run. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct BuildContext { + pub out_dir: PathBuf, + pub sys_dir: PathBuf, + pub target_os: TargetOs, + pub target_triple: String, + pub host_triple: String, + pub cargo_cfg_target_arch: String, + pub cargo_cfg_target_env: String, + pub build_shared_libs: bool, + pub profile: String, + pub opt_level: String, + pub debug_info: String, + pub include_debug_messages: bool, +} + +impl BuildContext { + /// # Errors + /// + /// Returns [`BuildError::Environment`] for a variable cargo sets for build scripts that is + /// missing or not Unicode, or the [`BuildError`] of [`TargetOs::detect`] when the target + /// cannot be built for. + pub fn from_environment( + environment: &BuildEnvironment, + host_os: HostOs, + ) -> Result { + let target_triple = environment.required("TARGET")?; + let cargo_cfg_target_arch = environment.required("CARGO_CFG_TARGET_ARCH")?; + + Ok(Self { + out_dir: PathBuf::from(environment.required("OUT_DIR")?), + sys_dir: PathBuf::from(environment.required("CARGO_MANIFEST_DIR")?), + target_os: TargetOs::detect( + environment, + &target_triple, + &cargo_cfg_target_arch, + host_os, + )?, + host_triple: environment.required("HOST")?, + cargo_cfg_target_env: environment + .optional("CARGO_CFG_TARGET_ENV")? + .unwrap_or_default(), + build_shared_libs: environment.has(BuildFeature::DynamicLink), + profile: native_profile(&environment.required("PROFILE")?), + opt_level: environment.required("OPT_LEVEL")?, + debug_info: environment.required("DEBUG")?, + include_debug_messages: environment.optional("BUILD_DEBUG")?.is_some(), + target_triple, + cargo_cfg_target_arch, + }) + } + + #[must_use] + pub fn cmake_dir(&self) -> PathBuf { + self.out_dir.join("cmake") + } + + #[must_use] + pub fn llama_src(&self) -> PathBuf { + self.sys_dir.join("llama.cpp") + } + + #[must_use] + pub fn debug_messages(&self) -> Vec { + [ + format!("TARGET: {}", self.target_triple), + format!("CARGO_MANIFEST_DIR: {}", self.sys_dir.display()), + format!("OUT_DIR: {}", self.out_dir.display()), + format!("CMAKE_DIR: {}", self.cmake_dir().display()), + format!("BUILD_SHARED: {}", self.build_shared_libs), + ] + .map(CargoDirective::Debug) + .to_vec() + } +} + +#[cfg(test)] +pub mod tests { + use std::ffi::OsStr; + use std::ffi::OsString; + use std::path::PathBuf; + + use enumflags2::BitFlags; + + use super::BuildContext; + use super::native_profile; + use crate::android_ndk::tests::FIXTURE_ANDROID_NDK; + use crate::build_environment::BuildEnvironment; + use crate::build_environment::tests::environment_of; + use crate::build_environment::tests::with_non_unicode; + use crate::build_feature::BuildFeature; + use crate::cargo_directive::CargoDirective; + use crate::host_os::HostOs; + use crate::target_os::TargetOs; + + const CARGO_VARIABLES: [(&str, &str); 10] = [ + ("CARGO_CFG_TARGET_ARCH", "x86_64"), + ("CARGO_CFG_TARGET_ENV", "gnu"), + ("CARGO_CFG_TARGET_OS", "linux"), + ("CARGO_MANIFEST_DIR", "/sys"), + ("DEBUG", "false"), + ("HOST", "x86_64-unknown-linux-gnu"), + ("OPT_LEVEL", "3"), + ("OUT_DIR", "/out"), + ("PROFILE", "release"), + ("TARGET", "x86_64-unknown-linux-gnu"), + ]; + + pub const ANDROID_OVERRIDES: [(&str, &str); 4] = [ + ("ANDROID_NDK_HOME", FIXTURE_ANDROID_NDK), + ("CARGO_CFG_TARGET_ARCH", "aarch64"), + ("CARGO_CFG_TARGET_OS", "android"), + ("TARGET", "aarch64-linux-android"), + ]; + + /// The variables cargo sets for a build script building for x86-64 Linux, with `overrides`. + pub fn cargo_environment( + features: BitFlags, + overrides: &[(&str, &str)], + ) -> BuildEnvironment { + let mut environment = environment_of( + features, + CARGO_VARIABLES.map(|(name, value)| (name, OsString::from(value))), + ); + + environment.variables.extend( + overrides + .iter() + .map(|(name, value)| (OsString::from(name), OsString::from(value))), + ); + + environment + } + + pub fn context_of(environment: &BuildEnvironment) -> BuildContext { + BuildContext::from_environment(environment, HostOs::Linux).unwrap() + } + + #[test] + fn cargo_debug_and_release_profiles_both_use_optimized_native_code() { + assert_eq!(native_profile("debug"), "Release"); + assert_eq!(native_profile("release"), "Release"); + } + + #[test] + fn custom_cargo_profile_name_is_preserved_for_cmake() { + assert_eq!(native_profile("RelWithDebInfo"), "RelWithDebInfo"); + } + + #[test] + fn the_cargo_variables_and_features_describe_the_build() { + assert_eq!( + context_of(&cargo_environment( + BitFlags::from(BuildFeature::DynamicLink), + &[("BUILD_DEBUG", "")] + )), + BuildContext { + out_dir: PathBuf::from("/out"), + sys_dir: PathBuf::from("/sys"), + target_os: TargetOs::Linux, + target_triple: "x86_64-unknown-linux-gnu".to_owned(), + host_triple: "x86_64-unknown-linux-gnu".to_owned(), + cargo_cfg_target_arch: "x86_64".to_owned(), + cargo_cfg_target_env: "gnu".to_owned(), + build_shared_libs: true, + profile: "Release".to_owned(), + opt_level: "3".to_owned(), + debug_info: "false".to_owned(), + include_debug_messages: true, + } + ); + } + + #[test] + fn every_variable_cargo_sets_for_build_scripts_is_required() { + let without = |name: &str| { + let mut environment = cargo_environment(BitFlags::empty(), &[]); + + environment.variables.remove(OsStr::new(name)); + + BuildContext::from_environment(&environment, HostOs::Linux) + .err() + .map(|error| error.to_string()) + }; + let required = CARGO_VARIABLES + .map(|(name, _)| name) + .into_iter() + .filter(|name| *name != "CARGO_CFG_TARGET_ENV") + .collect::>(); + + assert_eq!( + required + .iter() + .map(|name| without(name)) + .collect::>>(), + required + .iter() + .map(|name| Some(format!( + "environment variable {name} could not be read: environment variable not \ + found" + ))) + .collect::>>() + ); + } + + #[test] + fn optional_variables_that_are_not_unicode_are_refused() { + assert_eq!( + ["CARGO_CFG_TARGET_ENV", "BUILD_DEBUG"].map(|name| { + BuildContext::from_environment( + &with_non_unicode(cargo_environment(BitFlags::empty(), &[]), name), + HostOs::Linux, + ) + .err() + .map(|error| error.to_string()) + }), + ["CARGO_CFG_TARGET_ENV", "BUILD_DEBUG"].map(|name| Some(format!( + "environment variable {name} could not be read: environment variable was not \ + valid unicode: \"\\xFF\"" + ))) + ); + } + + #[test] + fn debug_messages_describe_where_the_build_happens() { + assert_eq!( + context_of(&cargo_environment(BitFlags::empty(), &[])).debug_messages(), + [ + "TARGET: x86_64-unknown-linux-gnu", + "CARGO_MANIFEST_DIR: /sys", + "OUT_DIR: /out", + "CMAKE_DIR: /out/cmake", + "BUILD_SHARED: false", + ] + .map(|message| CargoDirective::Debug(message.to_owned())) + ); + } +} diff --git a/llama-cpp-bindings-build/src/build_environment.rs b/llama-cpp-bindings-build/src/build_environment.rs new file mode 100644 index 000000000..daa86d862 --- /dev/null +++ b/llama-cpp-bindings-build/src/build_environment.rs @@ -0,0 +1,184 @@ +use std::collections::BTreeMap; +use std::env; +use std::env::VarError; +use std::ffi::OsStr; +use std::ffi::OsString; +use std::path::PathBuf; + +use enumflags2::BitFlags; +use find_cuda_helper::find_cuda_lib_dirs; + +use crate::build_error::BuildError; +use crate::build_feature::BuildFeature; +use crate::enabled_build_features::enabled_build_features; + +/// Everything the native build reads from outside: the environment variables, the cargo features +/// this crate was compiled with, and the CUDA library directories installed on the host. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct BuildEnvironment { + pub variables: BTreeMap, + pub features: BitFlags, + pub cuda_library_dirs: Vec, +} + +impl BuildEnvironment { + /// The environment of the running build script. + #[must_use] + pub fn of_process() -> Self { + Self { + variables: env::vars_os().collect(), + features: enabled_build_features(), + cuda_library_dirs: find_cuda_lib_dirs(), + } + } + + /// # Errors + /// + /// Returns [`VarError::NotPresent`] for a variable the environment lacks, or + /// [`VarError::NotUnicode`] for one that is not Unicode. + pub fn var(&self, name: &'static str) -> Result { + self.variables + .get(OsStr::new(name)) + .cloned() + .ok_or(VarError::NotPresent)? + .into_string() + .map_err(VarError::NotUnicode) + } + + /// # Errors + /// + /// Returns [`BuildError::Environment`] when the variable is missing or not Unicode. + pub fn required(&self, name: &'static str) -> Result { + self.var(name) + .map_err(|source| BuildError::Environment { name, source }) + } + + /// # Errors + /// + /// Returns [`BuildError::Environment`] when the variable is set but not Unicode. + pub fn optional(&self, name: &'static str) -> Result, BuildError> { + match self.var(name) { + Ok(value) => Ok(Some(value)), + Err(VarError::NotPresent) => Ok(None), + Err(source) => Err(BuildError::Environment { name, source }), + } + } + + #[must_use] + pub fn has(&self, feature: BuildFeature) -> bool { + self.features.contains(feature) + } +} + +#[cfg(test)] +pub mod tests { + use std::collections::BTreeMap; + use std::ffi::OsString; + use std::os::unix::ffi::OsStringExt; + + use enumflags2::BitFlags; + use find_cuda_helper::find_cuda_lib_dirs; + + use super::BuildEnvironment; + use crate::build_feature::BuildFeature; + use crate::enabled_build_features::enabled_build_features; + + #[must_use] + pub fn environment_of( + features: BitFlags, + variables: [(&str, OsString); VARIABLES], + ) -> BuildEnvironment { + BuildEnvironment { + variables: variables + .into_iter() + .map(|(name, value)| (OsString::from(name), value)) + .collect::>(), + features, + cuda_library_dirs: Vec::new(), + } + } + + #[must_use] + pub fn with_non_unicode(mut environment: BuildEnvironment, name: &str) -> BuildEnvironment { + environment + .variables + .insert(OsString::from(name), OsString::from_vec(vec![0xff])); + + environment + } + + #[test] + fn the_process_environment_holds_the_variables_features_and_cuda_lookup_of_the_process() { + let environment = BuildEnvironment::of_process(); + + assert_eq!( + environment.var("CARGO_MANIFEST_DIR"), + Ok(env!("CARGO_MANIFEST_DIR").to_owned()) + ); + assert_eq!(environment.features, enabled_build_features()); + assert_eq!(environment.cuda_library_dirs, find_cuda_lib_dirs()); + } + + #[test] + fn a_unicode_variable_is_read() { + let environment = environment_of( + BitFlags::empty(), + [("TARGET", OsString::from("x86_64-unknown-linux-gnu"))], + ); + + assert_eq!( + [ + environment.required("TARGET").ok(), + environment.optional("TARGET").ok().flatten() + ], + [(); 2].map(|()| Some("x86_64-unknown-linux-gnu".to_owned())) + ); + } + + #[test] + fn a_missing_variable_is_required_but_optional() { + let environment = environment_of(BitFlags::empty(), []); + + assert_eq!(environment.optional("HOST").ok(), Some(None)); + assert_eq!( + environment + .required("HOST") + .err() + .map(|error| error.to_string()), + Some( + "environment variable HOST could not be read: environment variable not found" + .to_owned() + ) + ); + } + + #[test] + fn a_variable_that_is_not_unicode_is_refused_even_when_optional() { + let environment = with_non_unicode(environment_of(BitFlags::empty(), []), "TARGET"); + + assert_eq!( + environment + .optional("TARGET") + .err() + .map(|error| error.to_string()), + Some( + "environment variable TARGET could not be read: environment variable was not \ + valid unicode: \"\\xFF\"" + .to_owned() + ) + ); + } + + #[test] + fn features_are_reported_by_membership() { + let environment = environment_of(BitFlags::from(BuildFeature::Cuda), []); + + assert_eq!( + [ + environment.has(BuildFeature::Cuda), + environment.has(BuildFeature::Metal) + ], + [true, false] + ); + } +} diff --git a/llama-cpp-bindings-build/src/build_error.rs b/llama-cpp-bindings-build/src/build_error.rs new file mode 100644 index 000000000..38c379a51 --- /dev/null +++ b/llama-cpp-bindings-build/src/build_error.rs @@ -0,0 +1,69 @@ +use std::env::VarError; +use std::io; +use std::path::PathBuf; + +#[derive(Debug, thiserror::Error)] +pub enum BuildError { + #[error("environment variable {name} could not be read: {source}")] + Environment { + name: &'static str, + #[source] + source: VarError, + }, + #[error("unsupported target operating system: {cargo_cfg_target_os}")] + UnsupportedTargetOs { cargo_cfg_target_os: String }, + #[error("Android NDK not found for target {target_triple}. Set ANDROID_NDK_HOME.")] + AndroidNdkRootNotConfigured { + target_triple: String, + #[source] + source: VarError, + }, + #[error("Android NDK path does not exist: {path}")] + AndroidNdkRootMissing { path: PathBuf }, + #[error("Android NDK toolchain file not found: {path}")] + AndroidNdkToolchainFileMissing { path: PathBuf }, + #[error("Android NDK toolchain not found at: {path}")] + AndroidNdkToolchainDirectoryMissing { path: PathBuf }, + #[error("Unsupported host platform for Android NDK")] + AndroidNdkUnsupportedHostPlatform, + #[error("Unsupported Android target architecture: {cargo_cfg_target_arch}")] + UnsupportedAndroidTarget { cargo_cfg_target_arch: String }, + #[error("ANDROID_PLATFORM is set but could not be read: {source}")] + AndroidPlatformUnreadable { + #[source] + source: VarError, + }, + #[error("Android NDK Clang directory could not be listed at {path}: {source}")] + AndroidNdkClangDirectoryUnreadable { + path: PathBuf, + #[source] + source: io::Error, + }, + #[error("Android NDK Clang version directory was not found at {path}")] + AndroidNdkClangVersionDirectoryMissing { path: PathBuf }, + #[error("Android NDK Clang built-in include directory was not found at {path}")] + AndroidNdkClangBuiltinIncludesMissing { path: PathBuf }, + #[error("bindgen failed: {0}")] + Bindgen(#[source] bindgen::BindgenError), + #[error( + "bindgen never saw field {type_name}::{field_name}, so it can no longer be kept private" + )] + PrivatizedFieldMissing { + type_name: &'static str, + field_name: &'static str, + }, + #[error("generated bindings could not be written: {0}")] + BindingsWrite(#[source] io::Error), + #[error("native wrapper compilation failed: {0}")] + NativeWrapper(#[source] cc::Error), + #[error("filesystem operation failed for {path}: {source}")] + Filesystem { + path: PathBuf, + #[source] + source: io::Error, + }, + #[error("environment path {name} does not name a directory: {path}")] + EnvironmentDirectory { name: &'static str, path: PathBuf }, + #[error("cargo directives could not be written: {0}")] + DirectiveWrite(#[source] io::Error), +} diff --git a/llama-cpp-bindings-build/src/build_feature.rs b/llama-cpp-bindings-build/src/build_feature.rs new file mode 100644 index 000000000..b690db85a --- /dev/null +++ b/llama-cpp-bindings-build/src/build_feature.rs @@ -0,0 +1,20 @@ +use enumflags2::bitflags; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[bitflags] +#[repr(u16)] +pub enum BuildFeature { + Cuda = 1 << 0, + CudaNoVmm = 1 << 1, + DynamicBackends = 1 << 2, + DynamicLink = 1 << 3, + FaultInjection = 1 << 4, + Metal = 1 << 5, + Openmp = 1 << 6, + Rocm = 1 << 7, + SharedStdcxx = 1 << 8, + StaticStdcxx = 1 << 9, + SystemGgml = 1 << 10, + SystemGgmlStatic = 1 << 11, + Vulkan = 1 << 12, +} diff --git a/llama-cpp-bindings-build/src/cargo_directive.rs b/llama-cpp-bindings-build/src/cargo_directive.rs new file mode 100644 index 000000000..0a16cb553 --- /dev/null +++ b/llama-cpp-bindings-build/src/cargo_directive.rs @@ -0,0 +1,99 @@ +use std::fmt; +use std::fmt::Display; +use std::fmt::Formatter; +use std::path::PathBuf; + +use crate::native_library::NativeLibrary; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum CargoDirective { + RerunIfChanged(PathBuf), + RerunIfEnvChanged(String), + LinkSearch(PathBuf), + LinkSearchNative(PathBuf), + LinkLibrary(NativeLibrary), + Metadata { key: &'static str, value: String }, + Debug(String), +} + +impl CargoDirective { + #[must_use] + pub const fn is_debug(&self) -> bool { + matches!(self, Self::Debug(_)) + } +} + +impl Display for CargoDirective { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::RerunIfChanged(path) => { + write!(formatter, "cargo:rerun-if-changed={}", path.display()) + } + Self::RerunIfEnvChanged(name) => write!(formatter, "cargo:rerun-if-env-changed={name}"), + Self::LinkSearch(path) => { + write!(formatter, "cargo:rustc-link-search={}", path.display()) + } + Self::LinkSearchNative(path) => { + write!( + formatter, + "cargo:rustc-link-search=native={}", + path.display() + ) + } + Self::LinkLibrary(library) => write!(formatter, "cargo:rustc-link-lib={library}"), + Self::Metadata { key, value } => write!(formatter, "cargo:{key}={value}"), + Self::Debug(message) => write!(formatter, "cargo:warning=[DEBUG] {message}"), + } + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::CargoDirective; + use crate::link_kind::LinkKind; + use crate::native_library::NativeLibrary; + + fn every_directive() -> [CargoDirective; 7] { + [ + CargoDirective::RerunIfChanged(PathBuf::from("wrapper.h")), + CargoDirective::RerunIfEnvChanged("CUDAARCHS".to_owned()), + CargoDirective::LinkSearch(PathBuf::from("/out/lib")), + CargoDirective::LinkSearchNative(PathBuf::from("/out/build/common")), + CargoDirective::LinkLibrary(NativeLibrary { + link_kind: LinkKind::Static, + name: "llama", + }), + CargoDirective::Metadata { + key: "backends_dir", + value: "/out/backends".to_owned(), + }, + CargoDirective::Debug("TARGET: x86_64-unknown-linux-gnu".to_owned()), + ] + } + + #[test] + fn every_directive_is_written_in_the_cargo_syntax() { + assert_eq!( + every_directive().map(|directive| directive.to_string()), + [ + "cargo:rerun-if-changed=wrapper.h", + "cargo:rerun-if-env-changed=CUDAARCHS", + "cargo:rustc-link-search=/out/lib", + "cargo:rustc-link-search=native=/out/build/common", + "cargo:rustc-link-lib=static=llama", + "cargo:backends_dir=/out/backends", + "cargo:warning=[DEBUG] TARGET: x86_64-unknown-linux-gnu", + ] + ); + } + + #[test] + fn only_debug_messages_are_debug_directives() { + assert_eq!( + every_directive().map(|directive| directive.is_debug()), + [false, false, false, false, false, false, true] + ); + } +} diff --git a/llama-cpp-bindings-build/src/cmake_build.rs b/llama-cpp-bindings-build/src/cmake_build.rs new file mode 100644 index 000000000..de4f8b193 --- /dev/null +++ b/llama-cpp-bindings-build/src/cmake_build.rs @@ -0,0 +1,60 @@ +use std::fs; +use std::path::PathBuf; + +use cmake::Config; + +use crate::build_error::BuildError; +use crate::cmake_define::CmakeDefine; + +/// Configures the `CMake` project in `source_dir` and builds and installs it into `out_dir`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CmakeBuild { + pub source_dir: PathBuf, + pub out_dir: PathBuf, + pub profile: String, + pub target_triple: String, + pub host_triple: String, + pub defines: Vec, + pub compiler_flags: Vec, + pub required_directories: Vec, +} + +impl CmakeBuild { + /// # Errors + /// + /// Returns [`BuildError::Filesystem`] when a directory the project installs into cannot be + /// created. + /// + /// # Panics + /// + /// Panics when `CMake` fails, the way the `cmake` crate reports a failed build. + pub fn build(&self) -> Result<(), BuildError> { + self.required_directories.iter().try_for_each(|directory| { + fs::create_dir_all(directory).map_err(|source| BuildError::Filesystem { + path: directory.clone(), + source, + }) + })?; + + let mut config = Config::new(&self.source_dir); + + for CmakeDefine { name, value } in &self.defines { + config.define(name, value); + } + + for compiler_flag in &self.compiler_flags { + config.cflag(compiler_flag); + config.cxxflag(compiler_flag); + } + + config + .target(&self.target_triple) + .host(&self.host_triple) + .out_dir(&self.out_dir) + .profile(&self.profile) + .always_configure(false) + .build(); + + Ok(()) + } +} diff --git a/llama-cpp-bindings-build/src/cmake_config.rs b/llama-cpp-bindings-build/src/cmake_config.rs deleted file mode 100644 index f688bde40..000000000 --- a/llama-cpp-bindings-build/src/cmake_config.rs +++ /dev/null @@ -1,368 +0,0 @@ -use std::env; -use std::path::{Path, PathBuf}; - -use cmake::Config; - -use crate::BuildContext; -use crate::BuildError; -use crate::android_ndk::AndroidNdk; -use crate::debug_log; -use crate::optional_env; -use crate::target_os::TargetOs; - -pub fn configure_and_build(context: &BuildContext) -> Result { - let mut config = Config::new(&context.llama_src); - - configure_base_defines(&mut config); - configure_cpu_features( - &mut config, - &context.cargo_cfg_target_arch, - context.target_os, - )?; - configure_shared_libs(&mut config, context.build_shared_libs); - configure_platform_specific( - &mut config, - context.target_os, - &context.target_triple, - context.android_ndk.as_ref(), - ); - configure_gpu_backends(&mut config, context.target_os)?; - configure_openmp(&mut config, context.target_os); - configure_system_ggml(&mut config)?; - let backends_dir = configure_dynamic_backends(&mut config, &context.cmake_dir)?; - - config - .out_dir(&context.cmake_dir) - .profile(&context.profile) - .always_configure(false); - - let install_dir = config.build(); - - if let Some(dir) = backends_dir { - println!("cargo:backends_dir={}", dir.display()); - } - - Ok(install_dir) -} - -fn configure_dynamic_backends( - config: &mut Config, - cmake_dir: &Path, -) -> Result, BuildError> { - if !cfg!(feature = "dynamic-backends") { - return Ok(None); - } - - let backends_dir = cmake_dir.join("backends"); - - std::fs::create_dir_all(&backends_dir).map_err(|source| BuildError::Filesystem { - path: backends_dir.clone(), - source, - })?; - - config.define("GGML_BACKEND_DL", "ON"); - config.define("GGML_CPU_ALL_VARIANTS", "ON"); - config.define("GGML_BACKEND_DIR", &backends_dir); - - Ok(Some(backends_dir)) -} - -fn configure_base_defines(config: &mut Config) { - config.define("LLAMA_BUILD_TESTS", "OFF"); - config.define("LLAMA_BUILD_EXAMPLES", "OFF"); - config.define("LLAMA_BUILD_SERVER", "OFF"); - config.define("LLAMA_BUILD_TOOLS", "OFF"); - config.define("LLAMA_BUILD_APP", "OFF"); - config.define("LLAMA_BUILD_COMMON", "ON"); - config.define("LLAMA_BUILD_MTMD", "ON"); - config.define("MTMD_VIDEO", "OFF"); - config.define("LLAMA_OPENSSL", "OFF"); -} - -fn codegen_options(encoded_rustflags: &str) -> Vec<&str> { - let mut options = Vec::new(); - let mut flags = encoded_rustflags.split('\x1f'); - - while let Some(flag) = flags.next() { - if flag == "-C" || flag == "--codegen" { - options.extend(flags.next()); - } else if let Some(option) = flag - .strip_prefix("--codegen=") - .or_else(|| flag.strip_prefix("-C")) - { - options.push(option); - } - } - - options -} - -fn target_cpu_of(encoded_rustflags: &str) -> Option { - codegen_options(encoded_rustflags) - .into_iter() - .rev() - .find_map(|option| option.strip_prefix("target-cpu=")) - .map(ToOwned::to_owned) -} - -fn configure_cpu_features( - config: &mut Config, - cargo_cfg_target_arch: &str, - target_os: TargetOs, -) -> Result<(), BuildError> { - let target_cpu = - optional_env("CARGO_ENCODED_RUSTFLAGS")?.and_then(|rustflags| target_cpu_of(&rustflags)); - - if target_cpu.as_deref() == Some("native") { - debug_log!("Detected target-cpu=native, compiling with GGML_NATIVE"); - config.define("GGML_NATIVE", "ON"); - - return Ok(()); - } - - config.define("GGML_NATIVE", "OFF"); - - if let Some(ref cpu) = target_cpu { - debug_log!("Setting baseline architecture: -march={}", cpu); - config.cflag(format!("-march={cpu}")); - config.cxxflag(format!("-march={cpu}")); - } - - let features = optional_env("CARGO_CFG_TARGET_FEATURE")?.unwrap_or_default(); - debug_log!("Compiling with target features: {}", features); - - for feature in features.split(',') { - if let Some(ggml_flag) = map_cpu_feature_to_ggml(feature) { - config.define(ggml_flag, "ON"); - } - } - - if cargo_cfg_target_arch == "aarch64" - && target_os == TargetOs::Linux - && target_cpu.as_deref() != Some("native") - { - config.define("GGML_CPU_ARM_ARCH", "armv8-a"); - } - - Ok(()) -} - -fn map_cpu_feature_to_ggml(feature: &str) -> Option<&'static str> { - match feature { - "avx" => Some("GGML_AVX"), - "avx2" => Some("GGML_AVX2"), - "avx512bf16" => Some("GGML_AVX512_BF16"), - "avx512vbmi" => Some("GGML_AVX512_VBMI"), - "avx512vnni" => Some("GGML_AVX512_VNNI"), - "avxvnni" => Some("GGML_AVX_VNNI"), - "bmi2" => Some("GGML_BMI2"), - "f16c" => Some("GGML_F16C"), - "fma" => Some("GGML_FMA"), - "sse4.2" => Some("GGML_SSE42"), - _ => None, - } -} - -fn configure_shared_libs(config: &mut Config, build_shared_libs: bool) { - config.define( - "BUILD_SHARED_LIBS", - if build_shared_libs { "ON" } else { "OFF" }, - ); -} - -fn configure_platform_specific( - config: &mut Config, - target_os: TargetOs, - target_triple: &str, - android_ndk: Option<&AndroidNdk>, -) { - match target_os { - TargetOs::Apple(_) => { - config.define("GGML_BLAS", "OFF"); - } - TargetOs::Android => { - if let Some(ndk) = android_ndk { - configure_android_cmake(config, ndk, target_triple); - } - } - TargetOs::Linux => {} - } -} - -fn configure_android_cmake(config: &mut Config, ndk: &AndroidNdk, _target_triple: &str) { - #[cfg(all(feature = "shared-stdcxx", feature = "static-stdcxx"))] - compile_error!("Features 'shared-stdcxx' and 'static-stdcxx' are mutually exclusive"); - - println!("cargo:rerun-if-env-changed=ANDROID_NDK_HOME"); - println!("cargo:rerun-if-env-changed=ANDROID_PLATFORM"); - - config.define("CMAKE_TOOLCHAIN_FILE", ndk.cmake_toolchain_file()); - config.define("ANDROID_PLATFORM", ndk.android_platform()); - config.define("ANDROID_ABI", ndk.abi); - - if cfg!(feature = "static-stdcxx") { - config.define("ANDROID_STL", "c++_static"); - } else if cfg!(feature = "shared-stdcxx") { - config.define("ANDROID_STL", "c++_shared"); - } - - config.define("GGML_LLAMAFILE", "OFF"); - - println!("cargo:rustc-link-lib=log"); - println!("cargo:rustc-link-lib=android"); -} - -fn configure_gpu_backends(config: &mut Config, target_os: TargetOs) -> Result<(), BuildError> { - if cfg!(feature = "vulkan") { - config.define("GGML_VULKAN", "ON"); - configure_vulkan_linking(target_os)?; - } - - if cfg!(feature = "cuda") { - config.define("GGML_CUDA", "ON"); - - if cfg!(feature = "cuda-no-vmm") { - config.define("GGML_CUDA_NO_VMM", "ON"); - } - - if let Some(cuda_architectures) = optional_env("CUDAARCHS")? { - config.define("CMAKE_CUDA_ARCHITECTURES", cuda_architectures); - } - } - - if cfg!(feature = "rocm") { - config.define("GGML_HIP", "ON"); - } - - Ok(()) -} - -fn configure_vulkan_linking(target_os: TargetOs) -> Result<(), BuildError> { - match target_os { - TargetOs::Linux => { - match env::var("VULKAN_SDK") { - Ok(vulkan_path) => { - let vulkan_lib_path = Path::new(&vulkan_path).join("lib"); - - println!("cargo:rustc-link-search={}", vulkan_lib_path.display()); - } - Err(env::VarError::NotPresent) => {} - Err(source) => { - return Err(BuildError::Environment { - name: "VULKAN_SDK", - source, - }); - } - } - - println!("cargo:rustc-link-lib=vulkan"); - } - TargetOs::Apple(_) | TargetOs::Android => {} - } - - Ok(()) -} - -fn configure_openmp(config: &mut Config, target_os: TargetOs) { - let openmp_enabled = cfg!(feature = "openmp") && !target_os.is_android(); - - config.define("GGML_OPENMP", if openmp_enabled { "ON" } else { "OFF" }); -} - -fn configure_system_ggml(config: &mut Config) -> Result<(), BuildError> { - if cfg!(feature = "system-ggml") { - println!("cargo:rerun-if-env-changed=GGML_DIR"); - let ggml_dir = env::var("GGML_DIR").map_err(|source| BuildError::Environment { - name: "GGML_DIR", - source, - })?; - config.define("LLAMA_USE_SYSTEM_GGML", "ON"); - config.define("GGML_DIR", ggml_dir); - } - - Ok(()) -} - -#[cfg(test)] -mod cpu_feature_mapping_tests { - use super::map_cpu_feature_to_ggml; - - #[test] - fn every_supported_rust_cpu_feature_maps_to_its_ggml_switch() { - assert_eq!(map_cpu_feature_to_ggml("avx"), Some("GGML_AVX")); - assert_eq!(map_cpu_feature_to_ggml("avx2"), Some("GGML_AVX2")); - assert_eq!( - map_cpu_feature_to_ggml("avx512bf16"), - Some("GGML_AVX512_BF16") - ); - assert_eq!( - map_cpu_feature_to_ggml("avx512vbmi"), - Some("GGML_AVX512_VBMI") - ); - assert_eq!( - map_cpu_feature_to_ggml("avx512vnni"), - Some("GGML_AVX512_VNNI") - ); - assert_eq!(map_cpu_feature_to_ggml("avxvnni"), Some("GGML_AVX_VNNI")); - assert_eq!(map_cpu_feature_to_ggml("bmi2"), Some("GGML_BMI2")); - assert_eq!(map_cpu_feature_to_ggml("f16c"), Some("GGML_F16C")); - assert_eq!(map_cpu_feature_to_ggml("fma"), Some("GGML_FMA")); - assert_eq!(map_cpu_feature_to_ggml("sse4.2"), Some("GGML_SSE42")); - } - - #[test] - fn unsupported_rust_cpu_feature_does_not_enable_a_ggml_switch() { - assert_eq!(map_cpu_feature_to_ggml("aes"), None); - assert_eq!(map_cpu_feature_to_ggml(""), None); - } -} - -#[cfg(test)] -mod target_cpu_of_tests { - use super::target_cpu_of; - - #[test] - fn an_attached_codegen_target_cpu_is_found() { - assert_eq!( - target_cpu_of("-Ctarget-cpu=native"), - Some("native".to_owned()) - ); - } - - #[test] - fn a_codegen_target_cpu_given_as_the_next_flag_is_found() { - assert_eq!( - [ - target_cpu_of("-C\x1ftarget-cpu=znver4"), - target_cpu_of("--codegen\x1ftarget-cpu=znver4"), - target_cpu_of("--codegen=target-cpu=znver4"), - ], - [ - Some("znver4".to_owned()), - Some("znver4".to_owned()), - Some("znver4".to_owned()), - ] - ); - } - - #[test] - fn the_last_target_cpu_wins_as_it_does_for_rustc() { - assert_eq!( - target_cpu_of("-Ctarget-cpu=haswell\x1f-Ctarget-cpu=znver4"), - Some("znver4".to_owned()) - ); - } - - #[test] - fn a_target_cpu_inside_another_codegen_option_is_not_the_target_cpu() { - assert_eq!(target_cpu_of("-Clink-arg=-Wl,target-cpu=znver4"), None); - } - - #[test] - fn rustflags_without_a_target_cpu_name_no_target_cpu() { - assert_eq!( - target_cpu_of("-Copt-level=3\x1f--cfg\x1ffeature=\"x\""), - None - ); - } -} diff --git a/llama-cpp-bindings-build/src/cmake_configuration.rs b/llama-cpp-bindings-build/src/cmake_configuration.rs new file mode 100644 index 000000000..863d18dff --- /dev/null +++ b/llama-cpp-bindings-build/src/cmake_configuration.rs @@ -0,0 +1,525 @@ +use std::path::PathBuf; + +use crate::android_ndk::AndroidNdk; +use crate::build_context::BuildContext; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::build_feature::BuildFeature; +use crate::cargo_directive::CargoDirective; +use crate::cmake_define::CmakeDefine; +use crate::cpu_tuning::CpuTuning; +use crate::link_kind::LinkKind; +use crate::native_library::NativeLibrary; +use crate::target_os::TargetOs; + +fn base_defines() -> Vec { + vec![ + CmakeDefine::new("LLAMA_BUILD_TESTS", "OFF"), + CmakeDefine::new("LLAMA_BUILD_EXAMPLES", "OFF"), + CmakeDefine::new("LLAMA_BUILD_SERVER", "OFF"), + CmakeDefine::new("LLAMA_BUILD_TOOLS", "OFF"), + CmakeDefine::new("LLAMA_BUILD_APP", "OFF"), + CmakeDefine::new("LLAMA_BUILD_COMMON", "ON"), + CmakeDefine::new("LLAMA_BUILD_MTMD", "ON"), + CmakeDefine::new("MTMD_VIDEO", "OFF"), + CmakeDefine::new("LLAMA_OPENSSL", "OFF"), + ] +} + +/// How llama.cpp is configured with `CMake` for this build, and the cargo directives that +/// configuration implies. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CmakeConfiguration { + pub defines: Vec, + pub compiler_flags: Vec, + pub required_directories: Vec, + pub directives: Vec, +} + +impl CmakeConfiguration { + /// # Errors + /// + /// Returns the [`BuildError`] of [`CpuTuning::of`], or [`BuildError::Environment`] when a + /// variable an enabled feature reads is not Unicode, or when `GGML_DIR` is missing for + /// `system-ggml`. + pub fn of(context: &BuildContext, environment: &BuildEnvironment) -> Result { + let CpuTuning { + defines: cpu_defines, + compiler_flags, + debug_messages, + } = CpuTuning::of(context, environment)?; + let mut configuration = Self { + defines: [base_defines(), cpu_defines].concat(), + compiler_flags, + required_directories: Vec::new(), + directives: debug_messages, + }; + + configuration.defines.push(CmakeDefine::switch( + "BUILD_SHARED_LIBS", + context.build_shared_libs, + )); + configuration.configure_platform(context, environment); + configuration.configure_gpu_backends(context, environment)?; + configuration.defines.push(CmakeDefine::switch( + "GGML_OPENMP", + environment.has(BuildFeature::Openmp) + && !matches!(context.target_os, TargetOs::Android(_)), + )); + configuration.configure_system_ggml(environment)?; + configuration.configure_dynamic_backends(context, environment); + + Ok(configuration) + } + + fn configure_platform(&mut self, context: &BuildContext, environment: &BuildEnvironment) { + match &context.target_os { + TargetOs::Apple => self.defines.push(CmakeDefine::new("GGML_BLAS", "OFF")), + TargetOs::Android(android_ndk) => self.configure_android(android_ndk, environment), + TargetOs::Linux => {} + } + } + + fn configure_android(&mut self, android_ndk: &AndroidNdk, environment: &BuildEnvironment) { + self.directives.extend([ + CargoDirective::RerunIfEnvChanged("ANDROID_NDK_HOME".to_owned()), + CargoDirective::RerunIfEnvChanged("ANDROID_PLATFORM".to_owned()), + ]); + self.defines.extend([ + CmakeDefine::new("CMAKE_TOOLCHAIN_FILE", android_ndk.cmake_toolchain_file()), + CmakeDefine::new("ANDROID_PLATFORM", android_ndk.android_platform()), + CmakeDefine::new("ANDROID_ABI", android_ndk.abi), + ]); + + if environment.has(BuildFeature::StaticStdcxx) { + self.defines + .push(CmakeDefine::new("ANDROID_STL", "c++_static")); + } else if environment.has(BuildFeature::SharedStdcxx) { + self.defines + .push(CmakeDefine::new("ANDROID_STL", "c++_shared")); + } + + self.defines.push(CmakeDefine::new("GGML_LLAMAFILE", "OFF")); + self.directives.extend(["log", "android"].map(|name| { + CargoDirective::LinkLibrary(NativeLibrary { + link_kind: LinkKind::Unspecified, + name, + }) + })); + } + + fn configure_gpu_backends( + &mut self, + context: &BuildContext, + environment: &BuildEnvironment, + ) -> Result<(), BuildError> { + if environment.has(BuildFeature::Vulkan) { + self.defines.push(CmakeDefine::new("GGML_VULKAN", "ON")); + + if context.target_os == TargetOs::Linux { + self.link_vulkan(environment)?; + } + } + + if environment.has(BuildFeature::Cuda) { + self.defines.push(CmakeDefine::new("GGML_CUDA", "ON")); + + if environment.has(BuildFeature::CudaNoVmm) { + self.defines + .push(CmakeDefine::new("GGML_CUDA_NO_VMM", "ON")); + } + + if let Some(cuda_architectures) = environment.optional("CUDAARCHS")? { + self.defines.push(CmakeDefine::new( + "CMAKE_CUDA_ARCHITECTURES", + cuda_architectures, + )); + } + } + + if environment.has(BuildFeature::Rocm) { + self.defines.push(CmakeDefine::new("GGML_HIP", "ON")); + } + + Ok(()) + } + + fn link_vulkan(&mut self, environment: &BuildEnvironment) -> Result<(), BuildError> { + self.directives + .push(CargoDirective::RerunIfEnvChanged("VULKAN_SDK".to_owned())); + + if let Some(vulkan_sdk) = environment.optional("VULKAN_SDK")? { + self.directives.push(CargoDirective::LinkSearch( + PathBuf::from(vulkan_sdk).join("lib"), + )); + } + + self.directives + .push(CargoDirective::LinkLibrary(NativeLibrary { + link_kind: LinkKind::Unspecified, + name: "vulkan", + })); + + Ok(()) + } + + fn configure_system_ggml(&mut self, environment: &BuildEnvironment) -> Result<(), BuildError> { + if environment.has(BuildFeature::SystemGgml) { + self.directives + .push(CargoDirective::RerunIfEnvChanged("GGML_DIR".to_owned())); + + let ggml_dir = environment.required("GGML_DIR")?; + + self.defines.extend([ + CmakeDefine::new("LLAMA_USE_SYSTEM_GGML", "ON"), + CmakeDefine::new("GGML_DIR", ggml_dir), + ]); + } + + Ok(()) + } + + fn configure_dynamic_backends( + &mut self, + context: &BuildContext, + environment: &BuildEnvironment, + ) { + if environment.has(BuildFeature::DynamicBackends) { + let backends_dir = context.cmake_dir().join("backends"); + + self.defines.extend([ + CmakeDefine::new("GGML_BACKEND_DL", "ON"), + CmakeDefine::new("GGML_CPU_ALL_VARIANTS", "ON"), + CmakeDefine::new("GGML_BACKEND_DIR", backends_dir.clone()), + ]); + self.directives.push(CargoDirective::Metadata { + key: "backends_dir", + value: backends_dir.display().to_string(), + }); + self.required_directories.push(backends_dir); + } + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use enumflags2::BitFlags; + + use super::CmakeConfiguration; + use crate::android_ndk::tests::FIXTURE_ANDROID_NDK; + use crate::build_context::tests::ANDROID_OVERRIDES; + use crate::build_context::tests::cargo_environment; + use crate::build_context::tests::context_of; + use crate::build_environment::tests::with_non_unicode; + use crate::build_feature::BuildFeature; + use crate::cargo_directive::CargoDirective; + use crate::cmake_define::CmakeDefine; + use crate::link_kind::LinkKind; + use crate::native_library::NativeLibrary; + + fn configured( + features: BitFlags, + overrides: &[(&str, &str)], + ) -> CmakeConfiguration { + let environment = cargo_environment(features, overrides); + + CmakeConfiguration::of(&context_of(&environment), &environment).unwrap() + } + + fn after_the_base_switches( + features: BitFlags, + overrides: &[(&str, &str)], + ) -> CmakeConfiguration { + let mut configuration = configured(features, overrides); + + configuration.defines.drain(..9); + + configuration + } + + fn compiling_with_no_target_features() -> CargoDirective { + CargoDirective::Debug("Compiling with target features: ".to_owned()) + } + + fn unspecified_link(name: &'static str) -> CargoDirective { + CargoDirective::LinkLibrary(NativeLibrary { + link_kind: LinkKind::Unspecified, + name, + }) + } + + #[test] + fn a_plain_linux_build_turns_off_everything_but_the_libraries_the_bindings_use() { + assert_eq!( + configured(BitFlags::empty(), &[]), + CmakeConfiguration { + defines: vec![ + CmakeDefine::new("LLAMA_BUILD_TESTS", "OFF"), + CmakeDefine::new("LLAMA_BUILD_EXAMPLES", "OFF"), + CmakeDefine::new("LLAMA_BUILD_SERVER", "OFF"), + CmakeDefine::new("LLAMA_BUILD_TOOLS", "OFF"), + CmakeDefine::new("LLAMA_BUILD_APP", "OFF"), + CmakeDefine::new("LLAMA_BUILD_COMMON", "ON"), + CmakeDefine::new("LLAMA_BUILD_MTMD", "ON"), + CmakeDefine::new("MTMD_VIDEO", "OFF"), + CmakeDefine::new("LLAMA_OPENSSL", "OFF"), + CmakeDefine::new("GGML_NATIVE", "OFF"), + CmakeDefine::new("BUILD_SHARED_LIBS", "OFF"), + CmakeDefine::new("GGML_OPENMP", "OFF"), + ], + compiler_flags: Vec::new(), + required_directories: Vec::new(), + directives: vec![compiling_with_no_target_features()], + } + ); + } + + #[test] + fn apple_targets_build_without_blas_and_leave_vulkan_linking_to_the_platform() { + assert_eq!( + after_the_base_switches( + BuildFeature::Metal | BuildFeature::Vulkan, + &[("CARGO_CFG_TARGET_OS", "macos")] + ), + CmakeConfiguration { + defines: vec![ + CmakeDefine::new("GGML_NATIVE", "OFF"), + CmakeDefine::new("BUILD_SHARED_LIBS", "OFF"), + CmakeDefine::new("GGML_BLAS", "OFF"), + CmakeDefine::new("GGML_VULKAN", "ON"), + CmakeDefine::new("GGML_OPENMP", "OFF"), + ], + compiler_flags: Vec::new(), + required_directories: Vec::new(), + directives: vec![compiling_with_no_target_features()], + } + ); + } + + #[test] + fn android_targets_build_with_the_ndk_toolchain_their_stl_and_no_openmp() { + let android_with = |features: BitFlags| { + after_the_base_switches(features | BuildFeature::Openmp, &ANDROID_OVERRIDES) + }; + let android_defines = |stl: Vec| { + [ + vec![ + CmakeDefine::new("GGML_NATIVE", "OFF"), + CmakeDefine::new("BUILD_SHARED_LIBS", "OFF"), + CmakeDefine::new( + "CMAKE_TOOLCHAIN_FILE", + format!("{FIXTURE_ANDROID_NDK}/build/cmake/android.toolchain.cmake"), + ), + CmakeDefine::new("ANDROID_PLATFORM", "android-28"), + CmakeDefine::new("ANDROID_ABI", "arm64-v8a"), + ], + stl, + vec![ + CmakeDefine::new("GGML_LLAMAFILE", "OFF"), + CmakeDefine::new("GGML_OPENMP", "OFF"), + ], + ] + .concat() + }; + let android_configuration = |stl: Vec| CmakeConfiguration { + defines: android_defines(stl), + compiler_flags: Vec::new(), + required_directories: Vec::new(), + directives: vec![ + compiling_with_no_target_features(), + CargoDirective::RerunIfEnvChanged("ANDROID_NDK_HOME".to_owned()), + CargoDirective::RerunIfEnvChanged("ANDROID_PLATFORM".to_owned()), + unspecified_link("log"), + unspecified_link("android"), + ], + }; + + assert_eq!( + [ + android_with(BitFlags::from(BuildFeature::StaticStdcxx)), + android_with(BitFlags::from(BuildFeature::SharedStdcxx)), + android_with(BitFlags::empty()), + ], + [ + android_configuration(vec![CmakeDefine::new("ANDROID_STL", "c++_static")]), + android_configuration(vec![CmakeDefine::new("ANDROID_STL", "c++_shared")]), + android_configuration(Vec::new()), + ] + ); + } + + #[test] + fn gpu_backends_turn_on_their_ggml_switches_and_take_their_sdk_and_architectures_when_set() { + let gpu_configuration = |overrides: &[(&str, &str)]| { + after_the_base_switches( + BuildFeature::Vulkan + | BuildFeature::Cuda + | BuildFeature::CudaNoVmm + | BuildFeature::Rocm, + overrides, + ) + }; + let gpu_defines = |cuda_architectures: Vec| { + [ + vec![ + CmakeDefine::new("GGML_NATIVE", "OFF"), + CmakeDefine::new("BUILD_SHARED_LIBS", "OFF"), + CmakeDefine::new("GGML_VULKAN", "ON"), + CmakeDefine::new("GGML_CUDA", "ON"), + CmakeDefine::new("GGML_CUDA_NO_VMM", "ON"), + ], + cuda_architectures, + vec![ + CmakeDefine::new("GGML_HIP", "ON"), + CmakeDefine::new("GGML_OPENMP", "OFF"), + ], + ] + .concat() + }; + let vulkan_directives = |vulkan_sdk_search_path: Vec| { + [ + vec![ + compiling_with_no_target_features(), + CargoDirective::RerunIfEnvChanged("VULKAN_SDK".to_owned()), + ], + vulkan_sdk_search_path, + vec![unspecified_link("vulkan")], + ] + .concat() + }; + + assert_eq!( + [ + gpu_configuration(&[("VULKAN_SDK", "/vulkan"), ("CUDAARCHS", "89")]), + gpu_configuration(&[]), + ], + [ + CmakeConfiguration { + defines: gpu_defines(vec![CmakeDefine::new("CMAKE_CUDA_ARCHITECTURES", "89")]), + compiler_flags: Vec::new(), + required_directories: Vec::new(), + directives: vulkan_directives(vec![CargoDirective::LinkSearch(PathBuf::from( + "/vulkan/lib" + ))]), + }, + CmakeConfiguration { + defines: gpu_defines(Vec::new()), + compiler_flags: Vec::new(), + required_directories: Vec::new(), + directives: vulkan_directives(Vec::new()), + }, + ] + ); + } + + #[test] + fn openmp_is_turned_on_for_linux() { + assert_eq!( + after_the_base_switches(BitFlags::from(BuildFeature::Openmp), &[]).defines, + [ + CmakeDefine::new("GGML_NATIVE", "OFF"), + CmakeDefine::new("BUILD_SHARED_LIBS", "OFF"), + CmakeDefine::new("GGML_OPENMP", "ON"), + ] + ); + } + + #[test] + fn system_ggml_builds_llama_cpp_against_the_ggml_in_ggml_dir() { + assert_eq!( + after_the_base_switches( + BitFlags::from(BuildFeature::SystemGgml), + &[("GGML_DIR", "/ggml/lib/cmake/ggml")] + ), + CmakeConfiguration { + defines: vec![ + CmakeDefine::new("GGML_NATIVE", "OFF"), + CmakeDefine::new("BUILD_SHARED_LIBS", "OFF"), + CmakeDefine::new("GGML_OPENMP", "OFF"), + CmakeDefine::new("LLAMA_USE_SYSTEM_GGML", "ON"), + CmakeDefine::new("GGML_DIR", "/ggml/lib/cmake/ggml"), + ], + compiler_flags: Vec::new(), + required_directories: Vec::new(), + directives: vec![ + compiling_with_no_target_features(), + CargoDirective::RerunIfEnvChanged("GGML_DIR".to_owned()), + ], + } + ); + } + + #[test] + fn dynamic_backends_are_built_as_shared_libraries_installed_into_a_published_backends_dir() { + assert_eq!( + after_the_base_switches( + BuildFeature::DynamicBackends | BuildFeature::DynamicLink, + &[] + ), + CmakeConfiguration { + defines: vec![ + CmakeDefine::new("GGML_NATIVE", "OFF"), + CmakeDefine::new("BUILD_SHARED_LIBS", "ON"), + CmakeDefine::new("GGML_OPENMP", "OFF"), + CmakeDefine::new("GGML_BACKEND_DL", "ON"), + CmakeDefine::new("GGML_CPU_ALL_VARIANTS", "ON"), + CmakeDefine::new("GGML_BACKEND_DIR", "/out/cmake/backends"), + ], + compiler_flags: Vec::new(), + required_directories: vec![PathBuf::from("/out/cmake/backends")], + directives: vec![ + compiling_with_no_target_features(), + CargoDirective::Metadata { + key: "backends_dir", + value: "/out/cmake/backends".to_owned(), + }, + ], + } + ); + } + + #[test] + fn variables_the_configuration_reads_must_be_present_and_unicode() { + let context = context_of(&cargo_environment(BitFlags::empty(), &[])); + let refusal = |features: BitFlags, name: &str| { + CmakeConfiguration::of( + &context, + &with_non_unicode(cargo_environment(features, &[]), name), + ) + .err() + .map(|error| error.to_string()) + }; + let not_unicode = |name: &str| { + Some(format!( + "environment variable {name} could not be read: environment variable was not \ + valid unicode: \"\\xFF\"" + )) + }; + + assert_eq!( + [ + refusal(BitFlags::empty(), "CARGO_ENCODED_RUSTFLAGS"), + refusal(BitFlags::from(BuildFeature::Vulkan), "VULKAN_SDK"), + refusal(BitFlags::from(BuildFeature::Cuda), "CUDAARCHS"), + CmakeConfiguration::of( + &context, + &cargo_environment(BitFlags::from(BuildFeature::SystemGgml), &[]) + ) + .err() + .map(|error| error.to_string()), + ], + [ + not_unicode("CARGO_ENCODED_RUSTFLAGS"), + not_unicode("VULKAN_SDK"), + not_unicode("CUDAARCHS"), + Some( + "environment variable GGML_DIR could not be read: environment variable not \ + found" + .to_owned() + ), + ] + ); + } +} diff --git a/llama-cpp-bindings-build/src/cmake_define.rs b/llama-cpp-bindings-build/src/cmake_define.rs new file mode 100644 index 000000000..47edb4656 --- /dev/null +++ b/llama-cpp-bindings-build/src/cmake_define.rs @@ -0,0 +1,22 @@ +use std::ffi::OsString; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CmakeDefine { + pub name: &'static str, + pub value: OsString, +} + +impl CmakeDefine { + #[must_use] + pub fn new(name: &'static str, value: impl Into) -> Self { + Self { + name, + value: value.into(), + } + } + + #[must_use] + pub fn switch(name: &'static str, enabled: bool) -> Self { + Self::new(name, if enabled { "ON" } else { "OFF" }) + } +} diff --git a/llama-cpp-bindings-build/src/cpp_wrapper.rs b/llama-cpp-bindings-build/src/cpp_wrapper.rs deleted file mode 100644 index 20cde9e59..000000000 --- a/llama-cpp-bindings-build/src/cpp_wrapper.rs +++ /dev/null @@ -1,42 +0,0 @@ -use std::path::Path; - -use llama_cpp_wrapper_sources::cpp_standard::CPP_STANDARD; -use llama_cpp_wrapper_sources::fault_injection_define::FAULT_INJECTION_DEFINE; -use llama_cpp_wrapper_sources::system_include_flags::system_include_flags; -use llama_cpp_wrapper_sources::wrapper_include_dirs::WRAPPER_INCLUDE_DIRS; -use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; - -use crate::BuildError; -use crate::target_os::TargetOs; - -pub fn compile_cpp_wrappers(target_os: TargetOs) -> Result<(), BuildError> { - let mut build = cc::Build::new(); - - build.cpp(true).pic(true); - - for include_dir in WRAPPER_INCLUDE_DIRS { - build.include(include_dir); - } - - for system_include_flag in system_include_flags(Path::new(".")) { - build.flag(system_include_flag); - } - - build.flag_if_supported(format!("-std={CPP_STANDARD}")); - - if cfg!(feature = "fault-injection") { - build.define(FAULT_INJECTION_DEFINE, None); - } - - for source in WRAPPER_SOURCES { - build.file(source); - } - - if target_os.is_android() && cfg!(feature = "static-stdcxx") { - build.cpp_link_stdlib(None); - } - - build - .try_compile("llama_cpp_bindings_sys_common_wrapper") - .map_err(BuildError::NativeWrapper) -} diff --git a/llama-cpp-bindings-build/src/cpu_tuning.rs b/llama-cpp-bindings-build/src/cpu_tuning.rs new file mode 100644 index 000000000..88ccbc336 --- /dev/null +++ b/llama-cpp-bindings-build/src/cpu_tuning.rs @@ -0,0 +1,294 @@ +use crate::build_context::BuildContext; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::cargo_directive::CargoDirective; +use crate::cmake_define::CmakeDefine; +use crate::target_os::TargetOs; + +fn codegen_options(encoded_rustflags: &str) -> Vec<&str> { + let mut options = Vec::new(); + let mut flags = encoded_rustflags.split('\x1f'); + + while let Some(flag) = flags.next() { + if flag == "-C" || flag == "--codegen" { + options.extend(flags.next()); + } else if let Some(option) = flag + .strip_prefix("--codegen=") + .or_else(|| flag.strip_prefix("-C")) + { + options.push(option); + } + } + + options +} + +fn target_cpu_of(encoded_rustflags: &str) -> Option { + codegen_options(encoded_rustflags) + .into_iter() + .rev() + .find_map(|option| option.strip_prefix("target-cpu=")) + .map(ToOwned::to_owned) +} + +fn map_cpu_feature_to_ggml(feature: &str) -> Option<&'static str> { + match feature { + "avx" => Some("GGML_AVX"), + "avx2" => Some("GGML_AVX2"), + "avx512bf16" => Some("GGML_AVX512_BF16"), + "avx512vbmi" => Some("GGML_AVX512_VBMI"), + "avx512vnni" => Some("GGML_AVX512_VNNI"), + "avxvnni" => Some("GGML_AVX_VNNI"), + "bmi2" => Some("GGML_BMI2"), + "f16c" => Some("GGML_F16C"), + "fma" => Some("GGML_FMA"), + "sse4.2" => Some("GGML_SSE42"), + _ => None, + } +} + +/// How ggml is compiled for the CPU rustc compiles for, and the debug messages explaining it. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CpuTuning { + pub defines: Vec, + pub compiler_flags: Vec, + pub debug_messages: Vec, +} + +impl CpuTuning { + /// # Errors + /// + /// Returns [`BuildError::Environment`] when the rustflags or the target features are not + /// Unicode. + pub fn of(context: &BuildContext, environment: &BuildEnvironment) -> Result { + let target_cpu = environment + .optional("CARGO_ENCODED_RUSTFLAGS")? + .and_then(|rustflags| target_cpu_of(&rustflags)); + + if target_cpu.as_deref() == Some("native") { + return Ok(Self { + defines: vec![CmakeDefine::new("GGML_NATIVE", "ON")], + compiler_flags: Vec::new(), + debug_messages: vec![CargoDirective::Debug( + "Detected target-cpu=native, compiling with GGML_NATIVE".to_owned(), + )], + }); + } + + let target_features = environment + .optional("CARGO_CFG_TARGET_FEATURE")? + .unwrap_or_default(); + let compiler_flags = target_cpu + .iter() + .map(|cpu| format!("-march={cpu}")) + .collect::>(); + let mut defines = vec![CmakeDefine::new("GGML_NATIVE", "OFF")]; + + defines.extend( + target_features + .split(',') + .filter_map(map_cpu_feature_to_ggml) + .map(|ggml_switch| CmakeDefine::new(ggml_switch, "ON")), + ); + + if context.cargo_cfg_target_arch == "aarch64" && context.target_os == TargetOs::Linux { + defines.push(CmakeDefine::new("GGML_CPU_ARM_ARCH", "armv8-a")); + } + + Ok(Self { + defines, + debug_messages: compiler_flags + .iter() + .map(|flag| format!("Setting baseline architecture: {flag}")) + .chain([format!("Compiling with target features: {target_features}")]) + .map(CargoDirective::Debug) + .collect(), + compiler_flags, + }) + } +} + +#[cfg(test)] +mod cpu_feature_mapping_tests { + use super::map_cpu_feature_to_ggml; + + #[test] + fn every_supported_rust_cpu_feature_maps_to_its_ggml_switch() { + assert_eq!(map_cpu_feature_to_ggml("avx"), Some("GGML_AVX")); + assert_eq!(map_cpu_feature_to_ggml("avx2"), Some("GGML_AVX2")); + assert_eq!( + map_cpu_feature_to_ggml("avx512bf16"), + Some("GGML_AVX512_BF16") + ); + assert_eq!( + map_cpu_feature_to_ggml("avx512vbmi"), + Some("GGML_AVX512_VBMI") + ); + assert_eq!( + map_cpu_feature_to_ggml("avx512vnni"), + Some("GGML_AVX512_VNNI") + ); + assert_eq!(map_cpu_feature_to_ggml("avxvnni"), Some("GGML_AVX_VNNI")); + assert_eq!(map_cpu_feature_to_ggml("bmi2"), Some("GGML_BMI2")); + assert_eq!(map_cpu_feature_to_ggml("f16c"), Some("GGML_F16C")); + assert_eq!(map_cpu_feature_to_ggml("fma"), Some("GGML_FMA")); + assert_eq!(map_cpu_feature_to_ggml("sse4.2"), Some("GGML_SSE42")); + } + + #[test] + fn unsupported_rust_cpu_feature_does_not_enable_a_ggml_switch() { + assert_eq!(map_cpu_feature_to_ggml("aes"), None); + assert_eq!(map_cpu_feature_to_ggml(""), None); + } +} + +#[cfg(test)] +mod target_cpu_of_tests { + use super::target_cpu_of; + + #[test] + fn an_attached_codegen_target_cpu_is_found() { + assert_eq!( + target_cpu_of("-Ctarget-cpu=native"), + Some("native".to_owned()) + ); + } + + #[test] + fn a_codegen_target_cpu_given_as_the_next_flag_is_found() { + assert_eq!( + [ + target_cpu_of("-C\x1ftarget-cpu=znver4"), + target_cpu_of("--codegen\x1ftarget-cpu=znver4"), + target_cpu_of("--codegen=target-cpu=znver4"), + ], + [ + Some("znver4".to_owned()), + Some("znver4".to_owned()), + Some("znver4".to_owned()), + ] + ); + } + + #[test] + fn the_last_target_cpu_wins_as_it_does_for_rustc() { + assert_eq!( + target_cpu_of("-Ctarget-cpu=haswell\x1f-Ctarget-cpu=znver4"), + Some("znver4".to_owned()) + ); + } + + #[test] + fn a_target_cpu_inside_another_codegen_option_is_not_the_target_cpu() { + assert_eq!(target_cpu_of("-Clink-arg=-Wl,target-cpu=znver4"), None); + } + + #[test] + fn rustflags_without_a_target_cpu_name_no_target_cpu() { + assert_eq!( + target_cpu_of("-Copt-level=3\x1f--cfg\x1ffeature=\"x\""), + None + ); + } +} + +#[cfg(test)] +mod cpu_tuning_tests { + use enumflags2::BitFlags; + + use super::CpuTuning; + use crate::build_context::tests::cargo_environment; + use crate::build_context::tests::context_of; + use crate::build_environment::tests::with_non_unicode; + use crate::cargo_directive::CargoDirective; + use crate::cmake_define::CmakeDefine; + + fn tuned(overrides: &[(&str, &str)]) -> CpuTuning { + let environment = cargo_environment(BitFlags::empty(), overrides); + + CpuTuning::of(&context_of(&environment), &environment).unwrap() + } + + #[test] + fn a_native_target_cpu_lets_ggml_detect_the_cpu() { + assert_eq!( + tuned(&[ + ("CARGO_ENCODED_RUSTFLAGS", "-Ctarget-cpu=native"), + ("CARGO_CFG_TARGET_FEATURE", "avx2"), + ]), + CpuTuning { + defines: vec![CmakeDefine::new("GGML_NATIVE", "ON")], + compiler_flags: Vec::new(), + debug_messages: vec![CargoDirective::Debug( + "Detected target-cpu=native, compiling with GGML_NATIVE".to_owned() + )], + } + ); + } + + #[test] + fn a_named_target_cpu_sets_the_baseline_and_target_features_enable_their_ggml_switches() { + assert_eq!( + tuned(&[ + ("CARGO_ENCODED_RUSTFLAGS", "-Ctarget-cpu=znver4"), + ("CARGO_CFG_TARGET_FEATURE", "aes,avx2,fma"), + ]), + CpuTuning { + defines: vec![ + CmakeDefine::new("GGML_NATIVE", "OFF"), + CmakeDefine::new("GGML_AVX2", "ON"), + CmakeDefine::new("GGML_FMA", "ON"), + ], + compiler_flags: vec!["-march=znver4".to_owned()], + debug_messages: vec![ + CargoDirective::Debug( + "Setting baseline architecture: -march=znver4".to_owned() + ), + CargoDirective::Debug( + "Compiling with target features: aes,avx2,fma".to_owned() + ), + ], + } + ); + } + + #[test] + fn aarch64_linux_without_a_target_cpu_builds_for_the_armv8_baseline() { + assert_eq!( + tuned(&[ + ("CARGO_CFG_TARGET_ARCH", "aarch64"), + ("TARGET", "aarch64-unknown-linux-gnu"), + ]), + CpuTuning { + defines: vec![ + CmakeDefine::new("GGML_NATIVE", "OFF"), + CmakeDefine::new("GGML_CPU_ARM_ARCH", "armv8-a"), + ], + compiler_flags: Vec::new(), + debug_messages: vec![CargoDirective::Debug( + "Compiling with target features: ".to_owned() + )], + } + ); + } + + #[test] + fn rustflags_or_target_features_that_are_not_unicode_are_refused() { + let context = context_of(&cargo_environment(BitFlags::empty(), &[])); + + assert_eq!( + ["CARGO_ENCODED_RUSTFLAGS", "CARGO_CFG_TARGET_FEATURE"].map(|name| { + CpuTuning::of( + &context, + &with_non_unicode(cargo_environment(BitFlags::empty(), &[]), name), + ) + .err() + .map(|error| error.to_string()) + }), + ["CARGO_ENCODED_RUSTFLAGS", "CARGO_CFG_TARGET_FEATURE"].map(|name| Some(format!( + "environment variable {name} could not be read: environment variable was not \ + valid unicode: \"\\xFF\"" + ))) + ); + } +} diff --git a/llama-cpp-bindings-build/src/emit_directives.rs b/llama-cpp-bindings-build/src/emit_directives.rs new file mode 100644 index 000000000..cc06f2b41 --- /dev/null +++ b/llama-cpp-bindings-build/src/emit_directives.rs @@ -0,0 +1,73 @@ +use std::io::Write; + +use crate::build_error::BuildError; +use crate::cargo_directive::CargoDirective; + +/// Writes `directives` for cargo, one per line, leaving the debug messages out unless +/// `include_debug` asks for them. +/// +/// # Errors +/// +/// Returns [`BuildError::DirectiveWrite`] when `output` refuses a line. +pub fn emit_directives( + directives: &[CargoDirective], + include_debug: bool, + output: &mut impl Write, +) -> Result<(), BuildError> { + directives + .iter() + .filter(|directive| include_debug || !directive.is_debug()) + .try_for_each(|directive| writeln!(output, "{directive}")) + .map_err(BuildError::DirectiveWrite) +} + +#[cfg(test)] +mod tests { + use std::fs::File; + use std::path::PathBuf; + + use super::emit_directives; + use crate::cargo_directive::CargoDirective; + + fn rebuild_and_debug_directives() -> [CargoDirective; 2] { + [ + CargoDirective::RerunIfChanged(PathBuf::from("wrapper.h")), + CargoDirective::Debug("Bindings Created".to_owned()), + ] + } + + fn emitted(include_debug: bool) -> String { + let mut output = Vec::new(); + + emit_directives(&rebuild_and_debug_directives(), include_debug, &mut output).unwrap(); + + String::from_utf8(output).unwrap() + } + + #[test] + fn debug_messages_are_left_out_unless_asked_for() { + assert_eq!( + [emitted(false), emitted(true)], + [ + "cargo:rerun-if-changed=wrapper.h\n".to_owned(), + "cargo:rerun-if-changed=wrapper.h\ncargo:warning=[DEBUG] Bindings Created\n" + .to_owned() + ] + ); + } + + #[test] + fn an_output_that_refuses_writes_is_reported() { + let mut read_only = File::open(concat!(env!("CARGO_MANIFEST_DIR"), "/Cargo.toml")).unwrap(); + + assert_eq!( + emit_directives(&rebuild_and_debug_directives(), false, &mut read_only) + .err() + .map(|error| error.to_string()), + Some( + "cargo directives could not be written: Bad file descriptor (os error 9)" + .to_owned() + ) + ); + } +} diff --git a/llama-cpp-bindings-build/src/enabled_build_features.rs b/llama-cpp-bindings-build/src/enabled_build_features.rs new file mode 100644 index 000000000..37a1d89ab --- /dev/null +++ b/llama-cpp-bindings-build/src/enabled_build_features.rs @@ -0,0 +1,81 @@ +use enumflags2::BitFlags; + +use crate::build_feature::BuildFeature; + +#[cfg(all(feature = "shared-stdcxx", feature = "static-stdcxx"))] +compile_error!("Features 'shared-stdcxx' and 'static-stdcxx' are mutually exclusive"); + +/// The cargo features this build crate was compiled with. +#[must_use] +pub fn enabled_build_features() -> BitFlags { + let mut features = BitFlags::empty(); + + features.set(BuildFeature::Cuda, cfg!(feature = "cuda")); + features.set(BuildFeature::CudaNoVmm, cfg!(feature = "cuda-no-vmm")); + features.set( + BuildFeature::DynamicBackends, + cfg!(feature = "dynamic-backends"), + ); + features.set(BuildFeature::DynamicLink, cfg!(feature = "dynamic-link")); + features.set( + BuildFeature::FaultInjection, + cfg!(feature = "fault-injection"), + ); + features.set(BuildFeature::Metal, cfg!(feature = "metal")); + features.set(BuildFeature::Openmp, cfg!(feature = "openmp")); + features.set(BuildFeature::Rocm, cfg!(feature = "rocm")); + features.set(BuildFeature::SharedStdcxx, cfg!(feature = "shared-stdcxx")); + features.set(BuildFeature::StaticStdcxx, cfg!(feature = "static-stdcxx")); + features.set(BuildFeature::SystemGgml, cfg!(feature = "system-ggml")); + features.set( + BuildFeature::SystemGgmlStatic, + cfg!(feature = "system-ggml-static"), + ); + features.set(BuildFeature::Vulkan, cfg!(feature = "vulkan")); + + features +} + +#[cfg(test)] +mod tests { + use super::enabled_build_features; + use crate::build_feature::BuildFeature; + + #[test] + fn every_build_feature_mirrors_its_cargo_feature() { + let features = enabled_build_features(); + + assert_eq!( + [ + features.contains(BuildFeature::Cuda), + features.contains(BuildFeature::CudaNoVmm), + features.contains(BuildFeature::DynamicBackends), + features.contains(BuildFeature::DynamicLink), + features.contains(BuildFeature::FaultInjection), + features.contains(BuildFeature::Metal), + features.contains(BuildFeature::Openmp), + features.contains(BuildFeature::Rocm), + features.contains(BuildFeature::SharedStdcxx), + features.contains(BuildFeature::StaticStdcxx), + features.contains(BuildFeature::SystemGgml), + features.contains(BuildFeature::SystemGgmlStatic), + features.contains(BuildFeature::Vulkan), + ], + [ + cfg!(feature = "cuda"), + cfg!(feature = "cuda-no-vmm"), + cfg!(feature = "dynamic-backends"), + cfg!(feature = "dynamic-link"), + cfg!(feature = "fault-injection"), + cfg!(feature = "metal"), + cfg!(feature = "openmp"), + cfg!(feature = "rocm"), + cfg!(feature = "shared-stdcxx"), + cfg!(feature = "static-stdcxx"), + cfg!(feature = "system-ggml"), + cfg!(feature = "system-ggml-static"), + cfg!(feature = "vulkan"), + ] + ); + } +} diff --git a/llama-cpp-bindings-build/src/host_os.rs b/llama-cpp-bindings-build/src/host_os.rs new file mode 100644 index 000000000..d5e4ef85a --- /dev/null +++ b/llama-cpp-bindings-build/src/host_os.rs @@ -0,0 +1,72 @@ +use std::env::consts::OS; + +use crate::build_error::BuildError; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum HostOs { + Linux, + MacOs, + Unsupported, +} + +impl HostOs { + /// The operating system the build script runs on, named by [`std::env::consts::OS`]. + #[must_use] + pub fn current() -> Self { + Self::named(OS) + } + + fn named(operating_system: &str) -> Self { + match operating_system { + "linux" => Self::Linux, + "macos" => Self::MacOs, + _ => Self::Unsupported, + } + } + + /// The directory name of the prebuilt Android NDK toolchain for this host. + /// + /// # Errors + /// + /// Returns [`BuildError::AndroidNdkUnsupportedHostPlatform`] for a host the NDK ships no + /// toolchain for. + pub const fn android_ndk_tag(self) -> Result<&'static str, BuildError> { + match self { + Self::Linux => Ok("linux-x86_64"), + Self::MacOs => Ok("darwin-x86_64"), + Self::Unsupported => Err(BuildError::AndroidNdkUnsupportedHostPlatform), + } + } +} + +#[cfg(test)] +mod tests { + use super::HostOs; + + #[test] + fn hosts_are_named_like_the_standard_library_names_them() { + assert_eq!( + ["linux", "macos", "windows"].map(HostOs::named), + [HostOs::Linux, HostOs::MacOs, HostOs::Unsupported] + ); + } + + #[test] + fn every_supported_host_names_its_prebuilt_ndk_toolchain() { + assert_eq!( + [HostOs::Linux, HostOs::MacOs].map(|host_os| host_os.android_ndk_tag().ok()), + [Some("linux-x86_64"), Some("darwin-x86_64")] + ); + } + + #[test] + fn an_unsupported_host_has_no_ndk_toolchain() { + assert_eq!( + HostOs::Unsupported + .android_ndk_tag() + .err() + .map(|error| error.to_string()), + Some("Unsupported host platform for Android NDK".to_owned()) + ); + } +} diff --git a/llama-cpp-bindings-build/src/lib.rs b/llama-cpp-bindings-build/src/lib.rs index a690ab144..64dc80f3b 100644 --- a/llama-cpp-bindings-build/src/lib.rs +++ b/llama-cpp-bindings-build/src/lib.rs @@ -2,191 +2,129 @@ not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic) )] + +mod android_architecture; mod android_ndk; -mod apple_variant; -mod bindgen_config; -mod cmake_config; -mod cpp_wrapper; -mod library_linking; +mod binding_callbacks; +mod bindings_generation; +mod build_context; +pub mod build_environment; +pub mod build_error; +pub mod build_feature; +mod cargo_directive; +mod cmake_build; +mod cmake_configuration; +mod cmake_define; +mod cpu_tuning; +mod emit_directives; +mod enabled_build_features; +mod host_os; +mod link_directives; +mod link_kind; +mod native_build; mod native_library; -mod rebuild_tracking; +mod privatized_field; +mod rebuild_triggers; mod target_os; +mod wrapper_compilation; -use std::env; -use std::path::{Path, PathBuf}; - -use android_ndk::AndroidNdk; -use target_os::TargetOs; - -#[derive(Debug, thiserror::Error)] -pub enum BuildError { - #[error("environment variable {name} could not be read: {source}")] - Environment { - name: &'static str, - #[source] - source: env::VarError, - }, - #[error("unsupported target operating system: {cargo_cfg_target_os}")] - UnsupportedTargetOs { cargo_cfg_target_os: String }, - #[error(transparent)] - AndroidNdk(#[from] android_ndk::AndroidNdkDetectionError), - #[error("bindgen failed: {0}")] - Bindgen(#[source] bindgen::BindgenError), - #[error( - "bindgen never saw field {type_name}::{field_name}, so it can no longer be kept private" - )] - PrivatizedFieldMissing { - type_name: &'static str, - field_name: &'static str, - }, - #[error("generated bindings could not be written: {0}")] - BindingsWrite(#[source] std::io::Error), - #[error("native wrapper compilation failed: {0}")] - NativeWrapper(#[source] cc::Error), - #[error("filesystem operation failed for {path}: {source}")] - Filesystem { - path: PathBuf, - #[source] - source: std::io::Error, - }, - #[error("environment path {name} does not name a directory: {path}")] - EnvironmentDirectory { name: &'static str, path: PathBuf }, -} - -fn required_env(name: &'static str) -> Result { - env::var(name).map_err(|source| BuildError::Environment { name, source }) -} - -fn optional_env(name: &'static str) -> Result, BuildError> { - match env::var(name) { - Ok(value) => Ok(Some(value)), - Err(env::VarError::NotPresent) => Ok(None), - Err(source) => Err(BuildError::Environment { name, source }), - } -} - -#[macro_export] -macro_rules! debug_log { - ($($arg:tt)*) => { - if std::env::var("BUILD_DEBUG").is_ok() { - println!("cargo:warning=[DEBUG] {}", format!($($arg)*)); - } - }; -} - -#[derive(Debug)] -pub struct BuildContext { - pub out_dir: PathBuf, - pub cmake_dir: PathBuf, - pub llama_src: PathBuf, - pub target_os: TargetOs, - pub target_triple: String, - pub cargo_cfg_target_arch: String, - pub cargo_cfg_target_env: String, - pub build_shared_libs: bool, - pub profile: String, - pub android_ndk: Option, -} - -impl BuildContext { - fn detect() -> Result { - let target_triple = required_env("TARGET")?; - let cargo_cfg_target_os = required_env("CARGO_CFG_TARGET_OS")?; - let cargo_cfg_target_env = optional_env("CARGO_CFG_TARGET_ENV")?.unwrap_or_default(); - let target_os = TargetOs::from_cargo_cfg(&cargo_cfg_target_os).ok_or_else(|| { - BuildError::UnsupportedTargetOs { - cargo_cfg_target_os: cargo_cfg_target_os.clone(), - } - })?; - let out_dir = PathBuf::from(required_env("OUT_DIR")?); - let manifest_dir = required_env("CARGO_MANIFEST_DIR")?; - let llama_src = Path::new(&manifest_dir).join("llama.cpp"); - - let build_shared_libs = cfg!(feature = "dynamic-link"); - let profile = native_profile(&required_env("PROFILE")?); +use std::io::Write; - let cargo_cfg_target_arch = required_env("CARGO_CFG_TARGET_ARCH")?; - let android_ndk = if target_os.is_android() { - Some(AndroidNdk::detect(&target_triple, &cargo_cfg_target_arch)?) - } else { - None - }; - - let cmake_dir = out_dir.join("cmake"); - - debug_log!("TARGET: {}", target_triple); - debug_log!("CARGO_MANIFEST_DIR: {}", manifest_dir); - debug_log!("OUT_DIR: {}", out_dir.display()); - debug_log!("CMAKE_DIR: {}", cmake_dir.display()); - debug_log!("BUILD_SHARED: {}", build_shared_libs); - - Ok(Self { - out_dir, - cmake_dir, - llama_src, - target_os, - target_triple, - cargo_cfg_target_arch, - cargo_cfg_target_env, - build_shared_libs, - profile, - android_ndk, - }) - } -} - -fn native_profile(cargo_profile: &str) -> String { - match cargo_profile { - "debug" | "release" => "Release".to_owned(), - other => other.to_owned(), - } -} +use crate::build_context::BuildContext; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::host_os::HostOs; +use crate::native_build::NativeBuild; +/// Generates the bindings, compiles the C++ wrappers, and builds llama.cpp for the build script +/// run `environment` describes, then writes the cargo directives that link them to `output`. +/// /// # Errors /// -/// Returns [`BuildError`] when the build environment cannot be read, the target is -/// unsupported, or any of the native build steps fail. -pub fn build() -> Result<(), BuildError> { - let context = BuildContext::detect()?; +/// Returns the [`BuildError`] of the first decision or build step that fails; nothing is written +/// to `output` then. +pub fn build(environment: &BuildEnvironment, output: &mut impl Write) -> Result<(), BuildError> { + let context = BuildContext::from_environment(environment, HostOs::current())?; - rebuild_tracking::register_rebuild_triggers(&context.llama_src); + NativeBuild::of(&context, environment)?.run(output) +} - bindgen_config::generate_bindings( - &context.llama_src, - &context.out_dir, - context.target_os, - &context.target_triple, - context.android_ndk.as_ref(), - )?; +#[cfg(test)] +mod tests { + use std::ffi::OsStr; - cpp_wrapper::compile_cpp_wrappers(context.target_os)?; + use enumflags2::BitFlags; - let build_dir = cmake_config::configure_and_build(&context)?; + use super::build; + use crate::build_context::tests::cargo_environment; + use crate::build_environment::BuildEnvironment; + use crate::build_feature::BuildFeature; - library_linking::link_libraries( - &context.cmake_dir, - &build_dir, - context.target_os, - &context.cargo_cfg_target_env, - context.build_shared_libs, - &context.profile, - )?; + const FIXTURE_CMAKE_PROJECT: &str = + concat!(env!("CARGO_MANIFEST_DIR"), "/fixtures/cmake_project"); - Ok(()) -} + fn failure(environment: &BuildEnvironment) -> [Option; 2] { + let mut output = Vec::new(); -#[cfg(test)] -mod build_context_value_tests { - use super::native_profile; - - #[test] - fn cargo_debug_and_release_profiles_both_use_optimized_native_code() { - assert_eq!(native_profile("debug"), "Release"); - assert_eq!(native_profile("release"), "Release"); + [ + build(environment, &mut output) + .err() + .map(|error| error.to_string()), + String::from_utf8(output).ok(), + ] } #[test] - fn custom_cargo_profile_name_is_preserved_for_cmake() { - assert_eq!(native_profile("RelWithDebInfo"), "RelWithDebInfo"); + fn a_failure_at_any_stage_is_reported_and_nothing_is_emitted() { + let mut without_target = cargo_environment(BitFlags::empty(), &[]); + + without_target.variables.remove(OsStr::new("TARGET")); + + assert_eq!( + [ + failure(&without_target), + failure(&cargo_environment( + BitFlags::from(BuildFeature::SystemGgml), + &[] + )), + failure(&cargo_environment(BitFlags::from(BuildFeature::Rocm), &[])), + failure(&cargo_environment( + BitFlags::empty(), + &[("CARGO_MANIFEST_DIR", FIXTURE_CMAKE_PROJECT)] + )), + ], + [ + [ + Some( + "environment variable TARGET could not be read: environment variable not \ + found" + .to_owned() + ), + Some(String::new()) + ], + [ + Some( + "environment variable GGML_DIR could not be read: environment variable \ + not found" + .to_owned() + ), + Some(String::new()) + ], + [ + Some( + "environment variable ROCM_PATH could not be read: environment variable \ + not found" + .to_owned() + ), + Some(String::new()) + ], + [ + Some(format!( + "bindgen failed: header '{FIXTURE_CMAKE_PROJECT}/wrapper.h' does not exist." + )), + Some(String::new()) + ], + ] + ); } } diff --git a/llama-cpp-bindings-build/src/library_linking.rs b/llama-cpp-bindings-build/src/library_linking.rs deleted file mode 100644 index c779bfc3d..000000000 --- a/llama-cpp-bindings-build/src/library_linking.rs +++ /dev/null @@ -1,381 +0,0 @@ -use std::env; -use std::path::Path; - -use crate::BuildError; -use crate::apple_variant::AppleVariant; -use crate::debug_log; -use crate::native_library::NativeLibrary; -use crate::target_os::TargetOs; - -pub fn link_libraries( - cmake_dir: &Path, - build_dir: &Path, - target_os: TargetOs, - cargo_cfg_target_env: &str, - build_shared_libs: bool, - profile: &str, -) -> Result<(), BuildError> { - emit_search_paths(cmake_dir, build_dir); - link_system_ggml_paths()?; - link_cmake_built_libraries(cmake_dir, build_shared_libs, profile); - link_cuda_libraries(build_shared_libs); - link_rocm_libraries(build_shared_libs)?; - link_openmp(cargo_cfg_target_env); - link_platform_system_libraries(target_os); - - Ok(()) -} - -fn emit_search_paths(cmake_dir: &Path, build_dir: &Path) { - println!( - "cargo:rustc-link-search={}", - cmake_dir.join("lib").display() - ); - println!( - "cargo:rustc-link-search={}", - cmake_dir.join("lib64").display() - ); - println!("cargo:rustc-link-search={}", build_dir.display()); -} - -fn link_system_ggml_paths() -> Result<(), BuildError> { - if !cfg!(feature = "system-ggml") { - return Ok(()); - } - - println!("cargo:rerun-if-env-changed=GGML_LIBRARY_DIR"); - let library_dir = env::var("GGML_LIBRARY_DIR").map_err(|source| BuildError::Environment { - name: "GGML_LIBRARY_DIR", - source, - })?; - let library_dir = Path::new(&library_dir); - if !library_dir.is_dir() { - return Err(BuildError::EnvironmentDirectory { - name: "GGML_LIBRARY_DIR", - path: library_dir.to_path_buf(), - }); - } - println!("cargo:rustc-link-search=native={}", library_dir.display()); - - Ok(()) -} - -fn link_cmake_built_libraries(cmake_dir: &Path, build_shared_libs: bool, profile: &str) { - emit_private_dependency_search_paths(cmake_dir, profile); - for NativeLibrary { link_kind, name } in native_libraries(build_shared_libs) { - let link = format!("cargo:rustc-link-lib={link_kind}={name}"); - debug_log!("LINK {link}"); - println!("{link}"); - } -} - -fn native_libraries(build_shared_libs: bool) -> Vec { - let cmake_kind = if build_shared_libs { "dylib" } else { "static" }; - let mut libraries = vec![NativeLibrary { - link_kind: cmake_kind, - name: "llama-common", - }]; - - if !build_shared_libs { - libraries.extend([ - NativeLibrary { - link_kind: "static", - name: "llama-common-base", - }, - NativeLibrary { - link_kind: "static", - name: "cpp-httplib", - }, - ]); - } - - libraries.push(NativeLibrary { - link_kind: cmake_kind, - name: "mtmd", - }); - if !build_shared_libs { - libraries.push(NativeLibrary { - link_kind: "static", - name: "vendor-hash", - }); - } - libraries.push(NativeLibrary { - link_kind: cmake_kind, - name: "llama", - }); - - if cfg!(feature = "system-ggml") { - let ggml_kind = if cfg!(feature = "system-ggml-static") { - "static" - } else { - "dylib" - }; - libraries.extend([ - NativeLibrary { - link_kind: ggml_kind, - name: "ggml-cpu", - }, - NativeLibrary { - link_kind: ggml_kind, - name: "ggml-base", - }, - NativeLibrary { - link_kind: ggml_kind, - name: "ggml", - }, - ]); - return libraries; - } - - if !cfg!(feature = "dynamic-backends") { - if cfg!(feature = "cuda") { - libraries.push(NativeLibrary { - link_kind: cmake_kind, - name: "ggml-cuda", - }); - } - if cfg!(feature = "metal") { - libraries.push(NativeLibrary { - link_kind: cmake_kind, - name: "ggml-metal", - }); - } - if cfg!(feature = "vulkan") { - libraries.push(NativeLibrary { - link_kind: cmake_kind, - name: "ggml-vulkan", - }); - } - if cfg!(feature = "rocm") { - libraries.push(NativeLibrary { - link_kind: cmake_kind, - name: "ggml-hip", - }); - } - } - - libraries.extend([ - NativeLibrary { - link_kind: cmake_kind, - name: "ggml-cpu", - }, - NativeLibrary { - link_kind: cmake_kind, - name: "ggml-base", - }, - NativeLibrary { - link_kind: cmake_kind, - name: "ggml", - }, - ]); - libraries -} - -fn emit_private_dependency_search_paths(cmake_dir: &Path, profile: &str) { - let common_lib_dir = cmake_dir.join("build").join("common"); - let httplib_dir = cmake_dir.join("build").join("vendor").join("cpp-httplib"); - let hash_dir = cmake_dir.join("build").join("vendor").join("hash"); - - emit_search_path_with_profile(&common_lib_dir, profile); - emit_search_path_with_profile(&httplib_dir, profile); - emit_search_path_with_profile(&hash_dir, profile); -} - -fn emit_search_path_with_profile(lib_dir: &Path, profile: &str) { - println!("cargo:rustc-link-search=native={}", lib_dir.display()); - - let profile_dir = lib_dir.join(profile); - - println!("cargo:rustc-link-search=native={}", profile_dir.display()); -} - -fn link_cuda_libraries(build_shared_libs: bool) { - if !cfg!(feature = "cuda") || build_shared_libs { - return; - } - - println!("cargo:rerun-if-env-changed=CUDA_PATH"); - - for lib_dir in find_cuda_helper::find_cuda_lib_dirs() { - println!("cargo:rustc-link-search=native={}", lib_dir.display()); - } - - println!("cargo:rustc-link-lib=static=cudart_static"); - println!("cargo:rustc-link-lib=static=cublas_static"); - println!("cargo:rustc-link-lib=static=cublasLt_static"); - - if !cfg!(feature = "cuda-no-vmm") { - println!("cargo:rustc-link-lib=cuda"); - } - - println!("cargo:rustc-link-lib=static=culibos"); -} - -fn link_rocm_libraries(build_shared_libs: bool) -> Result<(), BuildError> { - if !cfg!(feature = "rocm") || build_shared_libs { - return Ok(()); - } - - println!("cargo:rerun-if-env-changed=ROCM_PATH"); - let rocm_path = env::var("ROCM_PATH").map_err(|source| BuildError::Environment { - name: "ROCM_PATH", - source, - })?; - - let rocm_lib = Path::new(&rocm_path).join("lib"); - - if !rocm_lib.is_dir() { - return Err(BuildError::EnvironmentDirectory { - name: "ROCM_PATH", - path: rocm_lib, - }); - } - - println!("cargo:rustc-link-search=native={}", rocm_lib.display()); - println!("cargo:rustc-link-lib=dylib=amdhip64"); - println!("cargo:rustc-link-lib=dylib=rocblas"); - println!("cargo:rustc-link-lib=dylib=hipblas"); - - Ok(()) -} - -fn link_openmp(cargo_cfg_target_env: &str) { - if cfg!(feature = "openmp") && cargo_cfg_target_env == "gnu" { - println!("cargo:rustc-link-lib=gomp"); - } -} - -fn link_platform_system_libraries(target_os: TargetOs) { - match target_os { - TargetOs::Linux => { - println!("cargo:rustc-link-lib=dylib=stdc++"); - } - TargetOs::Apple(variant) => { - link_apple_frameworks(variant); - } - TargetOs::Android => { - link_android_cpp_stdlib(); - } - } -} - -fn link_android_cpp_stdlib() { - if cfg!(feature = "static-stdcxx") { - println!("cargo:rustc-link-lib=c++_static"); - println!("cargo:rustc-link-lib=c++abi"); - } else if cfg!(feature = "shared-stdcxx") { - println!("cargo:rustc-link-lib=c++_shared"); - } -} - -fn link_apple_frameworks(_variant: AppleVariant) { - println!("cargo:rustc-link-lib=framework=Foundation"); - println!("cargo:rustc-link-lib=framework=Metal"); - println!("cargo:rustc-link-lib=framework=MetalKit"); - println!("cargo:rustc-link-lib=framework=Accelerate"); - println!("cargo:rustc-link-lib=c++"); -} - -#[cfg(test)] -mod native_link_graph_tests { - #[cfg(feature = "cuda")] - use super::NativeLibrary; - use super::native_libraries; - - #[test] - fn the_cuda_backend_archive_is_linked_only_when_the_feature_is_enabled() { - let links_cuda_backend = native_libraries(false) - .iter() - .any(|library| library.name == "ggml-cuda"); - - assert_eq!(links_cuda_backend, cfg!(feature = "cuda")); - } - - #[cfg(feature = "cuda")] - #[test] - fn static_cuda_link_graph_contains_every_owned_archive_in_dependency_order() { - assert_eq!( - native_libraries(false), - vec![ - NativeLibrary { - link_kind: "static", - name: "llama-common", - }, - NativeLibrary { - link_kind: "static", - name: "llama-common-base", - }, - NativeLibrary { - link_kind: "static", - name: "cpp-httplib", - }, - NativeLibrary { - link_kind: "static", - name: "mtmd", - }, - NativeLibrary { - link_kind: "static", - name: "vendor-hash", - }, - NativeLibrary { - link_kind: "static", - name: "llama", - }, - NativeLibrary { - link_kind: "static", - name: "ggml-cuda", - }, - NativeLibrary { - link_kind: "static", - name: "ggml-cpu", - }, - NativeLibrary { - link_kind: "static", - name: "ggml-base", - }, - NativeLibrary { - link_kind: "static", - name: "ggml", - }, - ] - ); - } - - #[cfg(feature = "cuda")] - #[test] - fn dynamic_cuda_link_graph_uses_shared_top_level_libraries() { - assert_eq!( - native_libraries(true), - vec![ - NativeLibrary { - link_kind: "dylib", - name: "llama-common", - }, - NativeLibrary { - link_kind: "dylib", - name: "mtmd", - }, - NativeLibrary { - link_kind: "dylib", - name: "llama", - }, - NativeLibrary { - link_kind: "dylib", - name: "ggml-cuda", - }, - NativeLibrary { - link_kind: "dylib", - name: "ggml-cpu", - }, - NativeLibrary { - link_kind: "dylib", - name: "ggml-base", - }, - NativeLibrary { - link_kind: "dylib", - name: "ggml", - }, - ] - ); - } -} diff --git a/llama-cpp-bindings-build/src/link_directives.rs b/llama-cpp-bindings-build/src/link_directives.rs new file mode 100644 index 000000000..c9f9362c2 --- /dev/null +++ b/llama-cpp-bindings-build/src/link_directives.rs @@ -0,0 +1,663 @@ +use std::path::Path; +use std::path::PathBuf; + +use crate::build_context::BuildContext; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::build_feature::BuildFeature; +use crate::cargo_directive::CargoDirective; +use crate::link_kind::LinkKind; +use crate::native_library::NativeLibrary; +use crate::target_os::TargetOs; + +const fn link(link_kind: LinkKind, name: &'static str) -> CargoDirective { + CargoDirective::LinkLibrary(NativeLibrary { link_kind, name }) +} + +fn existing_directory(name: &'static str, path: PathBuf) -> Result { + if path.is_dir() { + Ok(path) + } else { + Err(BuildError::EnvironmentDirectory { name, path }) + } +} + +fn cmake_search_paths(context: &BuildContext) -> Vec { + let cmake_dir = context.cmake_dir(); + + vec![ + CargoDirective::LinkSearch(cmake_dir.join("lib")), + CargoDirective::LinkSearch(cmake_dir.join("lib64")), + CargoDirective::LinkSearch(cmake_dir), + ] +} + +fn system_ggml_search_paths( + environment: &BuildEnvironment, +) -> Result, BuildError> { + if !environment.has(BuildFeature::SystemGgml) { + return Ok(Vec::new()); + } + + let library_dir = existing_directory( + "GGML_LIBRARY_DIR", + PathBuf::from(environment.required("GGML_LIBRARY_DIR")?), + )?; + + Ok(vec![ + CargoDirective::RerunIfEnvChanged("GGML_LIBRARY_DIR".to_owned()), + CargoDirective::LinkSearchNative(library_dir), + ]) +} + +fn private_dependency_search_paths(context: &BuildContext) -> Vec { + let build_dir = context.cmake_dir().join("build"); + + [ + build_dir.join("common"), + build_dir.join("vendor").join("cpp-httplib"), + build_dir.join("vendor").join("hash"), + ] + .into_iter() + .flat_map(|library_dir| { + [ + CargoDirective::LinkSearchNative(library_dir.clone()), + CargoDirective::LinkSearchNative(library_dir.join(&context.profile)), + ] + }) + .collect() +} + +fn native_libraries(context: &BuildContext, environment: &BuildEnvironment) -> Vec { + let cmake_kind = if context.build_shared_libs { + LinkKind::Dylib + } else { + LinkKind::Static + }; + let built = |name| NativeLibrary { + link_kind: cmake_kind, + name, + }; + let bundled = |name| NativeLibrary { + link_kind: LinkKind::Static, + name, + }; + let mut libraries = vec![built("llama-common")]; + + if !context.build_shared_libs { + libraries.extend([bundled("llama-common-base"), bundled("cpp-httplib")]); + } + + libraries.push(built("mtmd")); + + if !context.build_shared_libs { + libraries.push(bundled("vendor-hash")); + } + + libraries.push(built("llama")); + + if environment.has(BuildFeature::SystemGgml) { + let ggml_kind = if environment.has(BuildFeature::SystemGgmlStatic) { + LinkKind::Static + } else { + LinkKind::Dylib + }; + + libraries.extend(["ggml-cpu", "ggml-base", "ggml"].map(|name| NativeLibrary { + link_kind: ggml_kind, + name, + })); + + return libraries; + } + + if !environment.has(BuildFeature::DynamicBackends) { + if environment.has(BuildFeature::Cuda) { + libraries.push(built("ggml-cuda")); + } + + if environment.has(BuildFeature::Metal) { + libraries.push(built("ggml-metal")); + } + + if environment.has(BuildFeature::Vulkan) { + libraries.push(built("ggml-vulkan")); + } + + if environment.has(BuildFeature::Rocm) { + libraries.push(built("ggml-hip")); + } + } + + libraries.extend(["ggml-cpu", "ggml-base", "ggml"].map(built)); + + libraries +} + +fn native_library_links( + context: &BuildContext, + environment: &BuildEnvironment, +) -> Vec { + native_libraries(context, environment) + .into_iter() + .flat_map(|library| { + let directive = CargoDirective::LinkLibrary(library); + + [ + CargoDirective::Debug(format!("LINK {directive}")), + directive, + ] + }) + .collect() +} + +fn cuda_links(context: &BuildContext, environment: &BuildEnvironment) -> Vec { + if !environment.has(BuildFeature::Cuda) || context.build_shared_libs { + return Vec::new(); + } + + let mut directives = vec![ + CargoDirective::RerunIfEnvChanged("CUDA_PATH".to_owned()), + CargoDirective::RerunIfEnvChanged("CUDA_LIBRARY_PATH".to_owned()), + ]; + + directives.extend( + environment + .cuda_library_dirs + .iter() + .cloned() + .map(CargoDirective::LinkSearchNative), + ); + directives.extend( + ["cudart_static", "cublas_static", "cublasLt_static"] + .map(|name| link(LinkKind::Static, name)), + ); + + if !environment.has(BuildFeature::CudaNoVmm) { + directives.push(link(LinkKind::Unspecified, "cuda")); + } + + directives.push(link(LinkKind::Static, "culibos")); + + directives +} + +fn rocm_links( + context: &BuildContext, + environment: &BuildEnvironment, +) -> Result, BuildError> { + if !environment.has(BuildFeature::Rocm) || context.build_shared_libs { + return Ok(Vec::new()); + } + + let rocm_lib = existing_directory( + "ROCM_PATH", + Path::new(&environment.required("ROCM_PATH")?).join("lib"), + )?; + + Ok(vec![ + CargoDirective::RerunIfEnvChanged("ROCM_PATH".to_owned()), + CargoDirective::LinkSearchNative(rocm_lib), + link(LinkKind::Dylib, "amdhip64"), + link(LinkKind::Dylib, "rocblas"), + link(LinkKind::Dylib, "hipblas"), + ]) +} + +fn openmp_links(context: &BuildContext, environment: &BuildEnvironment) -> Vec { + if environment.has(BuildFeature::Openmp) && context.cargo_cfg_target_env == "gnu" { + vec![link(LinkKind::Unspecified, "gomp")] + } else { + Vec::new() + } +} + +fn platform_links(context: &BuildContext, environment: &BuildEnvironment) -> Vec { + match context.target_os { + TargetOs::Linux => vec![link(LinkKind::Dylib, "stdc++")], + TargetOs::Apple => vec![ + link(LinkKind::Framework, "Foundation"), + link(LinkKind::Framework, "Metal"), + link(LinkKind::Framework, "MetalKit"), + link(LinkKind::Framework, "Accelerate"), + link(LinkKind::Unspecified, "c++"), + ], + TargetOs::Android(_) if environment.has(BuildFeature::StaticStdcxx) => vec![ + link(LinkKind::Unspecified, "c++_static"), + link(LinkKind::Unspecified, "c++abi"), + ], + TargetOs::Android(_) if environment.has(BuildFeature::SharedStdcxx) => { + vec![link(LinkKind::Unspecified, "c++_shared")] + } + TargetOs::Android(_) => Vec::new(), + } +} + +/// The search paths and libraries that link the native build into the bindings, in the order +/// the linker resolves them. +/// +/// # Errors +/// +/// Returns [`BuildError::Environment`] when a library directory an enabled feature needs is not +/// set, or [`BuildError::EnvironmentDirectory`] when it does not name a directory. +pub fn link_directives( + context: &BuildContext, + environment: &BuildEnvironment, +) -> Result, BuildError> { + Ok([ + cmake_search_paths(context), + system_ggml_search_paths(environment)?, + private_dependency_search_paths(context), + native_library_links(context, environment), + cuda_links(context, environment), + rocm_links(context, environment)?, + openmp_links(context, environment), + platform_links(context, environment), + ] + .concat()) +} + +#[cfg(test)] +mod tests { + use std::path::Path; + use std::path::PathBuf; + + use enumflags2::BitFlags; + + use super::link_directives; + use crate::build_context::tests::ANDROID_OVERRIDES; + use crate::build_context::tests::cargo_environment; + use crate::build_context::tests::context_of; + use crate::build_environment::BuildEnvironment; + use crate::build_feature::BuildFeature; + use crate::cargo_directive::CargoDirective; + use crate::link_kind::LinkKind; + use crate::native_library::NativeLibrary; + + const FIXTURES: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/fixtures"); + + fn linked(environment: &BuildEnvironment) -> Vec { + link_directives(&context_of(environment), environment).unwrap() + } + + fn libraries(directives: &[CargoDirective]) -> Vec { + directives + .iter() + .filter_map(|directive| match directive { + CargoDirective::LinkLibrary(library) => Some(*library), + _ => None, + }) + .collect() + } + + fn search_paths_and_rebuild_triggers(directives: &[CargoDirective]) -> Vec { + directives + .iter() + .filter(|directive| { + !directive.is_debug() && !matches!(directive, CargoDirective::LinkLibrary(_)) + }) + .cloned() + .collect() + } + + fn platform_libraries( + features: BitFlags, + overrides: &[(&str, &str)], + ) -> Vec { + let libraries = libraries(&linked(&cargo_environment(features, overrides))); + + libraries + .split(|library| library.name == "ggml") + .next_back() + .unwrap() + .to_vec() + } + + fn library(link_kind: LinkKind, name: &'static str) -> NativeLibrary { + NativeLibrary { link_kind, name } + } + + fn cmake_search_paths() -> Vec { + vec![ + CargoDirective::LinkSearch(PathBuf::from("/out/cmake/lib")), + CargoDirective::LinkSearch(PathBuf::from("/out/cmake/lib64")), + CargoDirective::LinkSearch(PathBuf::from("/out/cmake")), + ] + } + + fn private_dependency_search_paths() -> Vec { + [ + "/out/cmake/build/common", + "/out/cmake/build/common/Release", + "/out/cmake/build/vendor/cpp-httplib", + "/out/cmake/build/vendor/cpp-httplib/Release", + "/out/cmake/build/vendor/hash", + "/out/cmake/build/vendor/hash/Release", + ] + .map(|path| CargoDirective::LinkSearchNative(PathBuf::from(path))) + .to_vec() + } + + fn static_llama_libraries() -> Vec { + [ + "llama-common", + "llama-common-base", + "cpp-httplib", + "mtmd", + "vendor-hash", + "llama", + ] + .map(|name| library(LinkKind::Static, name)) + .to_vec() + } + + #[test] + fn a_static_linux_build_links_every_archive_after_its_dependents_and_then_libstdcxx() { + let static_archives = [ + static_llama_libraries(), + ["ggml-cpu", "ggml-base", "ggml"] + .map(|name| library(LinkKind::Static, name)) + .to_vec(), + ] + .concat() + .into_iter() + .flat_map(|archive| { + let directive = CargoDirective::LinkLibrary(archive); + + [ + CargoDirective::Debug(format!("LINK {directive}")), + directive, + ] + }) + .collect::>(); + + assert_eq!( + linked(&cargo_environment(BitFlags::empty(), &[])), + [ + cmake_search_paths(), + private_dependency_search_paths(), + static_archives, + vec![CargoDirective::LinkLibrary(library( + LinkKind::Dylib, + "stdc++" + ))], + ] + .concat() + ); + } + + #[test] + fn a_dynamic_build_links_shared_libraries_and_leaves_gpu_backends_to_runtime_loading() { + assert_eq!( + libraries(&linked(&cargo_environment( + BuildFeature::DynamicLink + | BuildFeature::DynamicBackends + | BuildFeature::Cuda + | BuildFeature::Rocm, + &[] + ))), + [ + "llama-common", + "mtmd", + "llama", + "ggml-cpu", + "ggml-base", + "ggml", + "stdc++" + ] + .map(|name| library(LinkKind::Dylib, name)) + ); + } + + #[test] + fn static_gpu_backends_link_their_archives_and_the_vendor_runtimes() { + let rocm_root = std::env::temp_dir().join(format!( + "llama-cpp-bindings-build-rocm-{}", + std::process::id() + )); + std::fs::create_dir_all(rocm_root.join("lib")).unwrap(); + let environment = BuildEnvironment { + cuda_library_dirs: vec![PathBuf::from("/cuda/lib64")], + ..cargo_environment( + BuildFeature::Cuda + | BuildFeature::Metal + | BuildFeature::Vulkan + | BuildFeature::Rocm, + &[("ROCM_PATH", rocm_root.to_str().unwrap())], + ) + }; + let directives = linked(&environment); + + assert_eq!( + libraries(&directives), + [ + static_llama_libraries(), + [ + "ggml-cuda", + "ggml-metal", + "ggml-vulkan", + "ggml-hip", + "ggml-cpu", + "ggml-base", + "ggml", + "cudart_static", + "cublas_static", + "cublasLt_static", + ] + .map(|name| library(LinkKind::Static, name)) + .to_vec(), + vec![ + library(LinkKind::Unspecified, "cuda"), + library(LinkKind::Static, "culibos"), + library(LinkKind::Dylib, "amdhip64"), + library(LinkKind::Dylib, "rocblas"), + library(LinkKind::Dylib, "hipblas"), + library(LinkKind::Dylib, "stdc++"), + ], + ] + .concat() + ); + assert_eq!( + search_paths_and_rebuild_triggers(&directives), + [ + cmake_search_paths(), + private_dependency_search_paths(), + vec![ + CargoDirective::RerunIfEnvChanged("CUDA_PATH".to_owned()), + CargoDirective::RerunIfEnvChanged("CUDA_LIBRARY_PATH".to_owned()), + CargoDirective::LinkSearchNative(PathBuf::from("/cuda/lib64")), + CargoDirective::RerunIfEnvChanged("ROCM_PATH".to_owned()), + CargoDirective::LinkSearchNative(rocm_root.join("lib")), + ], + ] + .concat() + ); + std::fs::remove_dir_all(rocm_root).unwrap(); + } + + #[test] + fn cuda_without_virtual_memory_management_does_not_link_the_driver() { + assert_eq!( + libraries(&linked(&cargo_environment( + BuildFeature::Cuda | BuildFeature::CudaNoVmm, + &[] + ))), + [ + static_llama_libraries(), + [ + "ggml-cuda", + "ggml-cpu", + "ggml-base", + "ggml", + "cudart_static", + "cublas_static", + "cublasLt_static", + "culibos", + ] + .map(|name| library(LinkKind::Static, name)) + .to_vec(), + vec![library(LinkKind::Dylib, "stdc++")], + ] + .concat() + ); + } + + #[test] + fn system_ggml_links_the_installed_ggml_instead_of_any_built_backend() { + let system_ggml_linked = |features: BitFlags| { + linked(&cargo_environment( + features | BuildFeature::SystemGgml | BuildFeature::Cuda, + &[("GGML_LIBRARY_DIR", FIXTURES)], + )) + }; + let linked_with_ggml_as = |ggml_kind: LinkKind| { + [ + static_llama_libraries(), + ["ggml-cpu", "ggml-base", "ggml"] + .map(|name| library(ggml_kind, name)) + .to_vec(), + ["cudart_static", "cublas_static", "cublasLt_static"] + .map(|name| library(LinkKind::Static, name)) + .to_vec(), + vec![ + library(LinkKind::Unspecified, "cuda"), + library(LinkKind::Static, "culibos"), + library(LinkKind::Dylib, "stdc++"), + ], + ] + .concat() + }; + let shared_ggml = system_ggml_linked(BitFlags::empty()); + + assert_eq!( + [ + libraries(&shared_ggml), + libraries(&system_ggml_linked(BitFlags::from( + BuildFeature::SystemGgmlStatic + ))), + ], + [ + linked_with_ggml_as(LinkKind::Dylib), + linked_with_ggml_as(LinkKind::Static) + ] + ); + assert_eq!( + search_paths_and_rebuild_triggers(&shared_ggml), + [ + cmake_search_paths(), + vec![ + CargoDirective::RerunIfEnvChanged("GGML_LIBRARY_DIR".to_owned()), + CargoDirective::LinkSearchNative(PathBuf::from(FIXTURES)), + ], + private_dependency_search_paths(), + vec![ + CargoDirective::RerunIfEnvChanged("CUDA_PATH".to_owned()), + CargoDirective::RerunIfEnvChanged("CUDA_LIBRARY_PATH".to_owned()), + ], + ] + .concat() + ); + } + + #[test] + fn openmp_links_libgomp_only_for_gnu_targets() { + assert_eq!( + ["gnu", "musl"].map(|cargo_cfg_target_env| { + platform_libraries( + BitFlags::from(BuildFeature::Openmp), + &[("CARGO_CFG_TARGET_ENV", cargo_cfg_target_env)], + ) + }), + [ + vec![ + library(LinkKind::Unspecified, "gomp"), + library(LinkKind::Dylib, "stdc++"), + ], + vec![library(LinkKind::Dylib, "stdc++")], + ] + ); + } + + #[test] + fn apple_and_android_targets_link_their_platform_cxx_runtime() { + assert_eq!( + [ + platform_libraries(BitFlags::empty(), &[("CARGO_CFG_TARGET_OS", "macos")]), + platform_libraries( + BitFlags::from(BuildFeature::StaticStdcxx), + &ANDROID_OVERRIDES + ), + platform_libraries( + BitFlags::from(BuildFeature::SharedStdcxx), + &ANDROID_OVERRIDES + ), + platform_libraries(BitFlags::empty(), &ANDROID_OVERRIDES), + ], + [ + vec![ + library(LinkKind::Framework, "Foundation"), + library(LinkKind::Framework, "Metal"), + library(LinkKind::Framework, "MetalKit"), + library(LinkKind::Framework, "Accelerate"), + library(LinkKind::Unspecified, "c++"), + ], + vec![ + library(LinkKind::Unspecified, "c++_static"), + library(LinkKind::Unspecified, "c++abi"), + ], + vec![library(LinkKind::Unspecified, "c++_shared")], + Vec::new(), + ] + ); + } + + #[test] + fn library_directories_an_enabled_feature_needs_must_exist() { + let missing_directory = std::env::temp_dir().join(format!( + "llama-cpp-bindings-build-missing-{}", + std::process::id() + )); + let refusal = |features: BitFlags, overrides: &[(&str, &str)]| { + let environment = cargo_environment(features, overrides); + + link_directives(&context_of(&environment), &environment) + .err() + .map(|error| error.to_string()) + }; + + assert_eq!( + [ + refusal(BitFlags::from(BuildFeature::SystemGgml), &[]), + refusal( + BitFlags::from(BuildFeature::SystemGgml), + &[("GGML_LIBRARY_DIR", missing_directory.to_str().unwrap())] + ), + refusal(BitFlags::from(BuildFeature::Rocm), &[]), + refusal( + BitFlags::from(BuildFeature::Rocm), + &[("ROCM_PATH", env!("CARGO_MANIFEST_DIR"))] + ), + ], + [ + Some( + "environment variable GGML_LIBRARY_DIR could not be read: environment \ + variable not found" + .to_owned() + ), + Some(format!( + "environment path GGML_LIBRARY_DIR does not name a directory: {}", + missing_directory.display() + )), + Some( + "environment variable ROCM_PATH could not be read: environment variable not \ + found" + .to_owned() + ), + Some(format!( + "environment path ROCM_PATH does not name a directory: {}", + Path::new(env!("CARGO_MANIFEST_DIR")).join("lib").display() + )), + ] + ); + } +} diff --git a/llama-cpp-bindings-build/src/link_kind.rs b/llama-cpp-bindings-build/src/link_kind.rs new file mode 100644 index 000000000..b3d6b4beb --- /dev/null +++ b/llama-cpp-bindings-build/src/link_kind.rs @@ -0,0 +1,7 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LinkKind { + Unspecified, + Static, + Dylib, + Framework, +} diff --git a/llama-cpp-bindings-build/src/native_build.rs b/llama-cpp-bindings-build/src/native_build.rs new file mode 100644 index 000000000..176495fcf --- /dev/null +++ b/llama-cpp-bindings-build/src/native_build.rs @@ -0,0 +1,243 @@ +use std::io::Write; + +use crate::bindings_generation::BindingsGeneration; +use crate::build_context::BuildContext; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::cargo_directive::CargoDirective; +use crate::cmake_build::CmakeBuild; +use crate::cmake_configuration::CmakeConfiguration; +use crate::emit_directives::emit_directives; +use crate::link_directives::link_directives; +use crate::rebuild_triggers::rebuild_triggers; +use crate::wrapper_compilation::WrapperCompilation; + +/// Every step of the native build, decided before any of them runs. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct NativeBuild { + pub bindings: BindingsGeneration, + pub wrappers: WrapperCompilation, + pub cmake: CmakeBuild, + pub directives: Vec, + pub include_debug_messages: bool, +} + +impl NativeBuild { + /// # Errors + /// + /// Returns the [`BuildError`] of [`CmakeConfiguration::of`] or [`link_directives`]. + pub fn of(context: &BuildContext, environment: &BuildEnvironment) -> Result { + let CmakeConfiguration { + defines, + compiler_flags, + required_directories, + directives: cmake_directives, + } = CmakeConfiguration::of(context, environment)?; + + Ok(Self { + bindings: BindingsGeneration::of(context, environment), + wrappers: WrapperCompilation::of(context, environment), + cmake: CmakeBuild { + source_dir: context.llama_src(), + out_dir: context.cmake_dir(), + profile: context.profile.clone(), + target_triple: context.target_triple.clone(), + host_triple: context.host_triple.clone(), + defines, + compiler_flags, + required_directories, + }, + directives: [ + context.debug_messages(), + rebuild_triggers(context), + cmake_directives, + link_directives(context, environment)?, + ] + .concat(), + include_debug_messages: context.include_debug_messages, + }) + } + + /// Generates the bindings, compiles the wrappers, builds llama.cpp, and only then writes the + /// cargo directives to `output`. + /// + /// # Errors + /// + /// Returns the [`BuildError`] of the first step that fails. + pub fn run(self, output: &mut impl Write) -> Result<(), BuildError> { + let bindings_directives = self.bindings.generate()?; + + self.wrappers.compile()?; + self.cmake.build()?; + + emit_directives( + &[self.directives, bindings_directives].concat(), + self.include_debug_messages, + output, + ) + } +} + +#[cfg(test)] +mod tests { + use std::path::Path; + use std::path::PathBuf; + + use super::NativeBuild; + use crate::bindings_generation::BindingsGeneration; + use crate::cargo_directive::CargoDirective; + use crate::cmake_build::CmakeBuild; + use crate::cmake_define::CmakeDefine; + use crate::link_kind::LinkKind; + use crate::native_library::NativeLibrary; + use crate::wrapper_compilation::WrapperCompilation; + + #[cfg(all(target_arch = "x86_64", target_os = "linux", target_env = "gnu"))] + const HOST_TRIPLE: &str = "x86_64-unknown-linux-gnu"; + #[cfg(all(target_arch = "aarch64", target_os = "linux", target_env = "gnu"))] + const HOST_TRIPLE: &str = "aarch64-unknown-linux-gnu"; + #[cfg(all(target_arch = "x86_64", target_os = "macos"))] + const HOST_TRIPLE: &str = "x86_64-apple-darwin"; + #[cfg(all(target_arch = "aarch64", target_os = "macos"))] + const HOST_TRIPLE: &str = "aarch64-apple-darwin"; + + const FIXTURES: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/fixtures"); + + const MANIFEST: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/Cargo.toml"); + + fn fixture_build(root: &Path) -> NativeBuild { + NativeBuild { + bindings: BindingsGeneration { + header: PathBuf::from(FIXTURES).join("bindings/wrapper.h"), + clang_args: Vec::new(), + output: root.join("bindings.rs"), + }, + wrappers: WrapperCompilation { + sources: vec![PathBuf::from(FIXTURES).join("wrapper/wrapper_fixture.cpp")], + include_dirs: vec![PathBuf::from(FIXTURES).join("wrapper")], + flags: vec![format!("-isystem{FIXTURES}/wrapper/vendor")], + cpp_standard_flag: "-std=c++17".to_owned(), + defines: vec!["WRAPPER_FIXTURE_DEFINE"], + cpp_stdlib_linked_separately: false, + target_triple: HOST_TRIPLE.to_owned(), + host_triple: HOST_TRIPLE.to_owned(), + opt_level: "0".to_owned(), + debug_info: "false".to_owned(), + out_dir: root.join("wrapper"), + }, + cmake: CmakeBuild { + source_dir: PathBuf::from(FIXTURES).join("cmake_project"), + out_dir: root.join("cmake"), + profile: "Release".to_owned(), + target_triple: HOST_TRIPLE.to_owned(), + host_triple: HOST_TRIPLE.to_owned(), + defines: vec![CmakeDefine::new("FIXTURE_LIBRARY_NAME", "fixture")], + compiler_flags: vec!["-DFIXTURE_VALUE=1".to_owned()], + required_directories: vec![root.join("cmake/backends")], + }, + directives: vec![ + CargoDirective::LinkSearchNative(root.join("cmake/lib")), + CargoDirective::LinkLibrary(NativeLibrary { + link_kind: LinkKind::Static, + name: "fixture", + }), + CargoDirective::Debug("fixture planned".to_owned()), + ], + include_debug_messages: false, + } + } + + fn build_root(name: &str) -> PathBuf { + let root = std::env::temp_dir().join(format!( + "llama-cpp-bindings-build-{name}-{}", + std::process::id() + )); + + std::fs::create_dir_all(&root).unwrap(); + + root + } + + #[test] + fn a_successful_build_writes_every_artifact_whoever_links_the_cxx_runtime_then_the_directives() + { + for cpp_stdlib_linked_separately in [false, true] { + let root = build_root(&format!("successful-{cpp_stdlib_linked_separately}")); + let mut native_build = fixture_build(&root); + let mut output = Vec::new(); + + native_build.wrappers.cpp_stdlib_linked_separately = cpp_stdlib_linked_separately; + native_build.run(&mut output).unwrap(); + + assert_eq!( + String::from_utf8(output).unwrap(), + format!( + "cargo:rustc-link-search=native={root}/cmake/lib\n\ + cargo:rustc-link-lib=static=fixture\n\ + cargo:rerun-if-env-changed=TARGET\n\ + cargo:rerun-if-env-changed=BINDGEN_EXTRA_CLANG_ARGS\n\ + cargo:rerun-if-changed={FIXTURES}/bindings/wrapper.h\n\ + cargo:rerun-if-changed={FIXTURES}/bindings/llama_types.h\n", + root = root.display() + ) + ); + assert_eq!( + [ + root.join("bindings.rs").is_file(), + root.join("wrapper/libllama_cpp_bindings_sys_common_wrapper.a") + .is_file(), + root.join("cmake/lib/libfixture.a").is_file(), + root.join("cmake/backends").is_dir(), + ], + [true; 4] + ); + std::fs::remove_dir_all(root).unwrap(); + } + } + + #[test] + fn a_failed_wrapper_compilation_stops_the_build_before_cmake_and_any_directive() { + let root = build_root("failed-wrappers"); + let mut native_build = fixture_build(&root); + let mut output = Vec::new(); + + native_build.wrappers.out_dir = PathBuf::from(MANIFEST); + + assert_eq!( + native_build + .run(&mut output) + .err() + .map(|error| error.to_string()), + Some( + "native wrapper compilation failed: IOError: File exists (os error 17)".to_owned() + ) + ); + assert_eq!( + [output.is_empty(), root.join("cmake").exists()], + [true, false] + ); + std::fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn a_cmake_build_that_cannot_create_its_directories_emits_no_directive() { + let root = build_root("failed-cmake"); + let mut native_build = fixture_build(&root); + let mut output = Vec::new(); + + native_build.cmake.required_directories = vec![PathBuf::from(MANIFEST).join("backends")]; + + assert_eq!( + native_build + .run(&mut output) + .err() + .map(|error| error.to_string()), + Some(format!( + "filesystem operation failed for {MANIFEST}/backends: Not a directory (os error \ + 20)" + )) + ); + assert_eq!(output, Vec::::new()); + std::fs::remove_dir_all(root).unwrap(); + } +} diff --git a/llama-cpp-bindings-build/src/native_library.rs b/llama-cpp-bindings-build/src/native_library.rs index 71ed20e29..298569942 100644 --- a/llama-cpp-bindings-build/src/native_library.rs +++ b/llama-cpp-bindings-build/src/native_library.rs @@ -1,5 +1,48 @@ +use std::fmt; +use std::fmt::Display; +use std::fmt::Formatter; + +use crate::link_kind::LinkKind; + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct NativeLibrary { - pub link_kind: &'static str, + pub link_kind: LinkKind, pub name: &'static str, } + +impl Display for NativeLibrary { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + let Self { link_kind, name } = self; + + match link_kind { + LinkKind::Unspecified => write!(formatter, "{name}"), + LinkKind::Static => write!(formatter, "static={name}"), + LinkKind::Dylib => write!(formatter, "dylib={name}"), + LinkKind::Framework => write!(formatter, "framework={name}"), + } + } +} + +#[cfg(test)] +mod tests { + use super::NativeLibrary; + use crate::link_kind::LinkKind; + + #[test] + fn every_link_kind_prefixes_the_library_the_way_cargo_expects() { + assert_eq!( + [ + LinkKind::Unspecified, + LinkKind::Static, + LinkKind::Dylib, + LinkKind::Framework + ] + .map(|link_kind| NativeLibrary { + link_kind, + name: "llama" + } + .to_string()), + ["llama", "static=llama", "dylib=llama", "framework=llama"] + ); + } +} diff --git a/llama-cpp-bindings-build/src/privatized_field.rs b/llama-cpp-bindings-build/src/privatized_field.rs new file mode 100644 index 000000000..d1a637b1e --- /dev/null +++ b/llama-cpp-bindings-build/src/privatized_field.rs @@ -0,0 +1,20 @@ +/// A field of a C struct that the generated bindings keep private, so callers go through the +/// typed API instead. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct PrivatizedField { + pub type_name: &'static str, + pub field_name: &'static str, +} + +impl PrivatizedField { + pub const ALL: &'static [Self] = &[ + Self { + type_name: "llama_context_params", + field_name: "defrag_thold", + }, + Self { + type_name: "mtmd_context_params", + field_name: "image_marker", + }, + ]; +} diff --git a/llama-cpp-bindings-build/src/rebuild_tracking.rs b/llama-cpp-bindings-build/src/rebuild_tracking.rs deleted file mode 100644 index cf5523592..000000000 --- a/llama-cpp-bindings-build/src/rebuild_tracking.rs +++ /dev/null @@ -1,15 +0,0 @@ -use std::path::Path; - -use llama_cpp_wrapper_sources::wrapper_headers::WRAPPER_HEADERS; -use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; - -pub fn register_rebuild_triggers(llama_src: &Path) { - println!("cargo:rerun-if-changed=build.rs"); - println!("cargo:rerun-if-env-changed=CUDAARCHS"); - - for path in WRAPPER_HEADERS.iter().chain(WRAPPER_SOURCES) { - println!("cargo:rerun-if-changed={path}"); - } - - println!("cargo:rerun-if-changed={}", llama_src.display()); -} diff --git a/llama-cpp-bindings-build/src/rebuild_triggers.rs b/llama-cpp-bindings-build/src/rebuild_triggers.rs new file mode 100644 index 000000000..62c89b76c --- /dev/null +++ b/llama-cpp-bindings-build/src/rebuild_triggers.rs @@ -0,0 +1,65 @@ +use llama_cpp_wrapper_sources::wrapper_headers::WRAPPER_HEADERS; +use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; + +use crate::build_context::BuildContext; +use crate::cargo_directive::CargoDirective; + +/// The files and variables whose change makes cargo run the native build again. +#[must_use] +pub fn rebuild_triggers(context: &BuildContext) -> Vec { + [ + CargoDirective::RerunIfChanged(context.sys_dir.join("build.rs")), + CargoDirective::RerunIfEnvChanged("CUDAARCHS".to_owned()), + ] + .into_iter() + .chain( + WRAPPER_HEADERS + .iter() + .chain(WRAPPER_SOURCES) + .map(|wrapper_file| CargoDirective::RerunIfChanged(context.sys_dir.join(wrapper_file))), + ) + .chain([CargoDirective::RerunIfChanged(context.llama_src())]) + .collect() +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use enumflags2::BitFlags; + use llama_cpp_wrapper_sources::wrapper_headers::WRAPPER_HEADERS; + use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; + + use super::rebuild_triggers; + use crate::build_context::tests::cargo_environment; + use crate::build_context::tests::context_of; + use crate::cargo_directive::CargoDirective; + + #[test] + fn the_build_script_cuda_architectures_every_wrapper_file_and_llama_cpp_trigger_a_rebuild() { + let triggers = rebuild_triggers(&context_of(&cargo_environment(BitFlags::empty(), &[]))); + + assert_eq!( + triggers, + [ + vec![ + CargoDirective::RerunIfChanged(PathBuf::from("/sys/build.rs")), + CargoDirective::RerunIfEnvChanged("CUDAARCHS".to_owned()), + ], + WRAPPER_HEADERS + .iter() + .chain(WRAPPER_SOURCES) + .map(|wrapper_file| { + CargoDirective::RerunIfChanged(PathBuf::from(format!( + "/sys/{wrapper_file}" + ))) + }) + .collect(), + vec![CargoDirective::RerunIfChanged(PathBuf::from( + "/sys/llama.cpp" + ))], + ] + .concat() + ); + } +} diff --git a/llama-cpp-bindings-build/src/target_os.rs b/llama-cpp-bindings-build/src/target_os.rs index 240a3de8d..7a012d06c 100644 --- a/llama-cpp-bindings-build/src/target_os.rs +++ b/llama-cpp-bindings-build/src/target_os.rs @@ -1,67 +1,133 @@ -use crate::apple_variant::AppleVariant; +use crate::android_ndk::AndroidNdk; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::host_os::HostOs; -#[derive(Debug, Clone, Copy, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq)] pub enum TargetOs { - Apple(AppleVariant), + Apple, Linux, - Android, + Android(AndroidNdk), } impl TargetOs { - #[must_use] - pub fn from_cargo_cfg(cargo_cfg_target_os: &str) -> Option { - match cargo_cfg_target_os { - "macos" => Some(Self::Apple(AppleVariant::MacOS)), - "ios" | "tvos" | "watchos" | "visionos" => Some(Self::Apple(AppleVariant::Other)), - "android" => Some(Self::Android), - "linux" => Some(Self::Linux), - _ => None, - } - } + /// The operating system cargo builds for, with the NDK it needs when that is Android. + /// + /// # Errors + /// + /// Returns [`BuildError::Environment`] when `CARGO_CFG_TARGET_OS` cannot be read, + /// [`BuildError::UnsupportedTargetOs`] for an operating system this crate does not build + /// for, or the [`BuildError`] of [`AndroidNdk::detect`] when an Android target has no + /// usable NDK. + pub fn detect( + environment: &BuildEnvironment, + target_triple: &str, + cargo_cfg_target_arch: &str, + host_os: HostOs, + ) -> Result { + let cargo_cfg_target_os = environment.required("CARGO_CFG_TARGET_OS")?; - #[must_use] - pub const fn is_android(self) -> bool { - matches!(self, Self::Android) + match cargo_cfg_target_os.as_str() { + "macos" | "ios" | "tvos" | "watchos" | "visionos" => Ok(Self::Apple), + "android" => Ok(Self::Android(AndroidNdk::detect( + environment, + target_triple, + cargo_cfg_target_arch, + host_os, + )?)), + "linux" => Ok(Self::Linux), + _ => Err(BuildError::UnsupportedTargetOs { + cargo_cfg_target_os, + }), + } } } #[cfg(test)] mod tests { + use std::ffi::OsString; + + use enumflags2::BitFlags; + use super::TargetOs; - use crate::apple_variant::AppleVariant; + use crate::android_ndk::AndroidNdk; + use crate::android_ndk::tests::FIXTURE_ANDROID_NDK; + use crate::build_environment::tests::environment_of; + use crate::host_os::HostOs; - #[test] - fn macos_is_distinguished_from_the_other_apple_platforms() { - assert_eq!( - TargetOs::from_cargo_cfg("macos"), - Some(TargetOs::Apple(AppleVariant::MacOS)) + fn detected(cargo_cfg_target_os: &str) -> Result { + let environment = environment_of( + BitFlags::empty(), + [ + ("CARGO_CFG_TARGET_OS", OsString::from(cargo_cfg_target_os)), + ("ANDROID_NDK_HOME", OsString::from(FIXTURE_ANDROID_NDK)), + ], ); - for apple_os in ["ios", "tvos", "watchos", "visionos"] { - assert_eq!( - TargetOs::from_cargo_cfg(apple_os), - Some(TargetOs::Apple(AppleVariant::Other)), - "{apple_os} must classify as a non-macOS Apple target" - ); - } + TargetOs::detect( + &environment, + "aarch64-linux-android", + "aarch64", + HostOs::Linux, + ) + .map_err(|error| error.to_string()) } #[test] - fn android_is_not_mistaken_for_linux() { - assert_eq!(TargetOs::from_cargo_cfg("android"), Some(TargetOs::Android)); - assert_eq!(TargetOs::from_cargo_cfg("linux"), Some(TargetOs::Linux)); + fn every_apple_platform_is_an_apple_target() { + assert_eq!( + ["macos", "ios", "tvos", "watchos", "visionos"].map(detected), + [(); 5].map(|()| Ok(TargetOs::Apple)) + ); } #[test] - fn only_android_needs_the_android_stdlib_handling() { - assert!(TargetOs::Android.is_android()); - assert!(!TargetOs::Linux.is_android()); + fn android_is_not_mistaken_for_linux_and_carries_its_ndk() { + let environment = environment_of( + BitFlags::empty(), + [("ANDROID_NDK_HOME", OsString::from(FIXTURE_ANDROID_NDK))], + ); + + assert_eq!( + [detected("android"), detected("linux")], + [ + Ok(TargetOs::Android( + AndroidNdk::detect( + &environment, + "aarch64-linux-android", + "aarch64", + HostOs::Linux + ) + .unwrap() + )), + Ok(TargetOs::Linux) + ] + ); } #[test] - fn an_unsupported_target_os_is_rejected() { - for unsupported_target_os in ["freebsd", "windows"] { - assert_eq!(TargetOs::from_cargo_cfg(unsupported_target_os), None); - } + fn an_unsupported_or_unusable_target_is_refused() { + assert_eq!( + [ + detected("freebsd"), + TargetOs::detect( + &environment_of( + BitFlags::empty(), + [("CARGO_CFG_TARGET_OS", "android".into())] + ), + "aarch64-linux-android", + "aarch64", + HostOs::Linux + ) + .map_err(|error| error.to_string()), + ], + [ + Err("unsupported target operating system: freebsd".to_owned()), + Err( + "Android NDK not found for target aarch64-linux-android. Set ANDROID_NDK_HOME." + .to_owned() + ), + ] + ); } } diff --git a/llama-cpp-bindings-build/src/wrapper_compilation.rs b/llama-cpp-bindings-build/src/wrapper_compilation.rs new file mode 100644 index 000000000..3e3355fd3 --- /dev/null +++ b/llama-cpp-bindings-build/src/wrapper_compilation.rs @@ -0,0 +1,158 @@ +use std::path::PathBuf; + +use llama_cpp_wrapper_sources::cpp_standard::CPP_STANDARD; +use llama_cpp_wrapper_sources::fault_injection_define::FAULT_INJECTION_DEFINE; +use llama_cpp_wrapper_sources::system_include_flags::system_include_flags; +use llama_cpp_wrapper_sources::wrapper_include_dirs::WRAPPER_INCLUDE_DIRS; +use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; + +use crate::build_context::BuildContext; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::build_feature::BuildFeature; +use crate::target_os::TargetOs; + +const WRAPPER_LIBRARY: &str = "llama_cpp_bindings_sys_common_wrapper"; + +/// Compiles the C++ wrapper sources into the static wrapper library in `out_dir`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct WrapperCompilation { + pub sources: Vec, + pub include_dirs: Vec, + pub flags: Vec, + pub cpp_standard_flag: String, + pub defines: Vec<&'static str>, + pub cpp_stdlib_linked_separately: bool, + pub target_triple: String, + pub host_triple: String, + pub opt_level: String, + pub debug_info: String, + pub out_dir: PathBuf, +} + +impl WrapperCompilation { + #[must_use] + pub fn of(context: &BuildContext, environment: &BuildEnvironment) -> Self { + Self { + sources: WRAPPER_SOURCES + .iter() + .map(|source| context.sys_dir.join(source)) + .collect(), + include_dirs: WRAPPER_INCLUDE_DIRS + .iter() + .map(|include_dir| context.sys_dir.join(include_dir)) + .collect(), + flags: system_include_flags(&context.sys_dir), + cpp_standard_flag: format!("-std={CPP_STANDARD}"), + defines: if environment.has(BuildFeature::FaultInjection) { + vec![FAULT_INJECTION_DEFINE] + } else { + Vec::new() + }, + cpp_stdlib_linked_separately: matches!(context.target_os, TargetOs::Android(_)) + && environment.has(BuildFeature::StaticStdcxx), + target_triple: context.target_triple.clone(), + host_triple: context.host_triple.clone(), + opt_level: context.opt_level.clone(), + debug_info: context.debug_info.clone(), + out_dir: context.out_dir.clone(), + } + } + + /// # Errors + /// + /// Returns [`BuildError::NativeWrapper`] when the wrapper library cannot be compiled. + pub fn compile(&self) -> Result<(), BuildError> { + let mut build = cc::Build::new(); + + build.cpp(true).pic(true).includes(&self.include_dirs); + + for flag in &self.flags { + build.flag(flag); + } + + build.flag_if_supported(&self.cpp_standard_flag); + + for define in &self.defines { + build.define(define, None); + } + + build.files(&self.sources); + + if self.cpp_stdlib_linked_separately { + build.cpp_link_stdlib(None); + } + + build + .target(&self.target_triple) + .host(&self.host_triple) + .opt_level_str(&self.opt_level) + .debug_str(&self.debug_info) + .out_dir(&self.out_dir) + .try_compile(WRAPPER_LIBRARY) + .map_err(BuildError::NativeWrapper) + } +} + +#[cfg(test)] +mod tests { + use std::path::Path; + use std::path::PathBuf; + + use enumflags2::BitFlags; + use llama_cpp_wrapper_sources::fault_injection_define::FAULT_INJECTION_DEFINE; + use llama_cpp_wrapper_sources::system_include_flags::system_include_flags; + use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; + + use super::WrapperCompilation; + use crate::build_context::tests::ANDROID_OVERRIDES; + use crate::build_context::tests::cargo_environment; + use crate::build_context::tests::context_of; + use crate::build_feature::BuildFeature; + + fn planned(features: BitFlags, overrides: &[(&str, &str)]) -> WrapperCompilation { + let environment = cargo_environment(features, overrides); + + WrapperCompilation::of(&context_of(&environment), &environment) + } + + fn linux_compilation() -> WrapperCompilation { + WrapperCompilation { + sources: WRAPPER_SOURCES + .iter() + .map(|source| PathBuf::from(format!("/sys/{source}"))) + .collect(), + include_dirs: vec![PathBuf::from("/sys/.")], + flags: system_include_flags(Path::new("/sys")), + cpp_standard_flag: "-std=c++17".to_owned(), + defines: Vec::new(), + cpp_stdlib_linked_separately: false, + target_triple: "x86_64-unknown-linux-gnu".to_owned(), + host_triple: "x86_64-unknown-linux-gnu".to_owned(), + opt_level: "3".to_owned(), + debug_info: "false".to_owned(), + out_dir: PathBuf::from("/out"), + } + } + + #[test] + fn the_wrappers_compile_as_cxx17_with_the_vendored_headers_as_system_includes() { + assert_eq!(planned(BitFlags::empty(), &[]), linux_compilation()); + } + + #[test] + fn fault_injection_defines_its_switch_and_a_static_android_stl_is_linked_separately() { + assert_eq!( + planned( + BuildFeature::FaultInjection | BuildFeature::StaticStdcxx, + &ANDROID_OVERRIDES + ), + WrapperCompilation { + defines: vec![FAULT_INJECTION_DEFINE], + cpp_stdlib_linked_separately: true, + target_triple: "aarch64-linux-android".to_owned(), + ..linux_compilation() + } + ); + } +} diff --git a/llama-cpp-bindings-sys/build.rs b/llama-cpp-bindings-sys/build.rs index bae50bed6..4d04a0415 100644 --- a/llama-cpp-bindings-sys/build.rs +++ b/llama-cpp-bindings-sys/build.rs @@ -1,3 +1,6 @@ -fn main() -> Result<(), llama_cpp_bindings_build::BuildError> { - llama_cpp_bindings_build::build() +use llama_cpp_bindings_build::build_environment::BuildEnvironment; +use llama_cpp_bindings_build::build_error::BuildError; + +fn main() -> Result<(), BuildError> { + llama_cpp_bindings_build::build(&BuildEnvironment::of_process(), &mut std::io::stdout()) } From 2368ad68116119ccca07245678df350efdcd4f6c Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Fri, 30 Oct 2026 22:14:51 +0100 Subject: [PATCH 18/82] Take the single clang version directory of the NDK toolchain instead of guessing it by name --- llama-cpp-bindings-build/src/android_ndk.rs | 57 ++++++++++++++++----- llama-cpp-bindings-build/src/build_error.rs | 7 +++ 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/llama-cpp-bindings-build/src/android_ndk.rs b/llama-cpp-bindings-build/src/android_ndk.rs index 0621d438b..c8083addd 100644 --- a/llama-cpp-bindings-build/src/android_ndk.rs +++ b/llama-cpp-bindings-build/src/android_ndk.rs @@ -55,9 +55,11 @@ fn detect_api_level(environment: &BuildEnvironment) -> Result Result { let clang_lib_path = PathBuf::from(toolchain_path).join("lib/clang"); - let version_dir = fs::read_dir(&clang_lib_path) + let mut version_dirs = fs::read_dir(&clang_lib_path) .and_then(Iterator::collect::, io::Error>>) .map_err(|source| BuildError::AndroidNdkClangDirectoryUnreadable { path: clang_lib_path.clone(), @@ -65,18 +67,23 @@ fn find_clang_builtin_includes(toolchain_path: &str) -> Result>(); + + version_dirs.sort(); + + let [version_dir] = <[PathBuf; 1]>::try_from(version_dirs).map_err(|candidates| { + if candidates.is_empty() { + BuildError::AndroidNdkClangVersionDirectoryMissing { + path: clang_lib_path.clone(), + } + } else { + BuildError::AndroidNdkClangVersionDirectoryAmbiguous { + path: clang_lib_path.clone(), + candidates, + } + } + })?; let include_path = version_dir.join("include"); if !include_path.is_dir() { @@ -381,7 +388,7 @@ pub mod tests { #[test] fn clang_builtin_include_resolution_requires_a_version_directory() { let toolchain = temporary_directory("clang-without-version"); - std::fs::create_dir_all(toolchain.join("lib/clang/not-a-version")).unwrap(); + std::fs::create_dir_all(toolchain.join("lib/clang")).unwrap(); std::fs::write(toolchain.join("lib/clang/21"), "").unwrap(); assert_eq!( @@ -396,6 +403,28 @@ pub mod tests { std::fs::remove_dir_all(toolchain).unwrap(); } + #[test] + fn clang_builtin_include_resolution_refuses_to_choose_between_version_directories() { + let toolchain = temporary_directory("clang-with-two-versions"); + std::fs::create_dir_all(toolchain.join("lib/clang/18/include")).unwrap(); + std::fs::create_dir_all(toolchain.join("lib/clang/17/include")).unwrap(); + + assert_eq!( + find_clang_builtin_includes(toolchain.to_str().unwrap()) + .err() + .map(|error| error.to_string()), + Some(format!( + "Android NDK Clang directory {} holds more than one version directory: {:?}", + toolchain.join("lib/clang").display(), + [ + toolchain.join("lib/clang/17"), + toolchain.join("lib/clang/18") + ] + )) + ); + std::fs::remove_dir_all(toolchain).unwrap(); + } + #[test] fn clang_builtin_include_resolution_requires_an_include_directory() { let toolchain = temporary_directory("clang-without-includes"); diff --git a/llama-cpp-bindings-build/src/build_error.rs b/llama-cpp-bindings-build/src/build_error.rs index 38c379a51..0c10d323e 100644 --- a/llama-cpp-bindings-build/src/build_error.rs +++ b/llama-cpp-bindings-build/src/build_error.rs @@ -41,6 +41,13 @@ pub enum BuildError { }, #[error("Android NDK Clang version directory was not found at {path}")] AndroidNdkClangVersionDirectoryMissing { path: PathBuf }, + #[error( + "Android NDK Clang directory {path} holds more than one version directory: {candidates:?}" + )] + AndroidNdkClangVersionDirectoryAmbiguous { + path: PathBuf, + candidates: Vec, + }, #[error("Android NDK Clang built-in include directory was not found at {path}")] AndroidNdkClangBuiltinIncludesMissing { path: PathBuf }, #[error("bindgen failed: {0}")] From 6b3298d100f9fe462d0ccae75c563a1c37a079fd Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Fri, 30 Oct 2026 22:14:51 +0100 Subject: [PATCH 19/82] Encode and decode through one llama_process wrapper with a single status mapping --- llama-cpp-bindings-sys/wrapper_common.cpp | 92 +-- llama-cpp-bindings-sys/wrapper_common.h | 48 +- .../tests/encoder_requirement.rs | 5 +- llama-cpp-bindings/src/context.rs | 641 +++++------------- llama-cpp-bindings/src/error.rs | 2 + llama-cpp-bindings/src/error/decode_error.rs | 21 +- llama-cpp-bindings/src/error/encode_error.rs | 21 +- .../src/error/process_failure.rs | 14 + llama-cpp-bindings/src/lib.rs | 4 +- llama-cpp-bindings/src/model.rs | 5 + 10 files changed, 238 insertions(+), 615 deletions(-) create mode 100644 llama-cpp-bindings/src/error/process_failure.rs diff --git a/llama-cpp-bindings-sys/wrapper_common.cpp b/llama-cpp-bindings-sys/wrapper_common.cpp index 134694b4c..50468c6ec 100644 --- a/llama-cpp-bindings-sys/wrapper_common.cpp +++ b/llama-cpp-bindings-sys/wrapper_common.cpp @@ -203,54 +203,6 @@ extern "C" auto llama_rs_memory_seq_pos_max( } } -extern "C" auto llama_rs_encode( - struct llama_context * ctx, - struct llama_batch_ext * batch, - int32_t * out_llama_cpp_return_code, - char ** out_error) -> llama_rs_encode_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - if (out_llama_cpp_return_code != nullptr) { - *out_llama_cpp_return_code = 0; - } - if (ctx == nullptr) { - return LLAMA_RS_ENCODE_NULL_CTX_ARG; - } - if (batch == nullptr) { - return LLAMA_RS_ENCODE_NULL_BATCH_ARG; - } - try { - const auto * model = llama_get_model(ctx); - if (model == nullptr) { - return LLAMA_RS_ENCODE_NULL_MODEL; - } - if (!llama_model_has_encoder(model)) { - return LLAMA_RS_ENCODE_MODEL_HAS_NO_ENCODER; - } - int32_t const result = llama_process(ctx, LLAMA_PROCESS_TYPE_ENCODE, batch); - if (result != 0) { - if (out_llama_cpp_return_code != nullptr) { - *out_llama_cpp_return_code = result; - } - if (result == -2) { - return LLAMA_RS_ENCODE_OUT_OF_MEMORY; - } - if (result == -3) { - return LLAMA_RS_ENCODE_COMPUTE_FAILED; - } - return LLAMA_RS_ENCODE_LLAMA_CPP_RETURNED_NONZERO_CODE; - } - return LLAMA_RS_ENCODE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_ENCODE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_ENCODE_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_ENCODE_ERROR_STRING_ALLOCATION_FAILED); - } -} - extern "C" auto llama_rs_memory_seq_add( const struct llama_context * ctx, llama_seq_id seq_id, @@ -454,47 +406,39 @@ extern "C" auto llama_rs_new_context_with_model( } } -extern "C" auto llama_rs_decode( +extern "C" auto llama_rs_process( struct llama_context * ctx, + enum llama_process_type process_type, struct llama_batch_ext * batch, int32_t * out_llama_cpp_return_code, - char ** out_error) -> llama_rs_decode_status { - if (out_error != nullptr) { - *out_error = nullptr; + char ** out_error) -> llama_rs_process_status { + if (out_error == nullptr) { + return LLAMA_RS_PROCESS_NULL_OUT_ERROR_ARG; } - if (out_llama_cpp_return_code != nullptr) { - *out_llama_cpp_return_code = 0; + *out_error = nullptr; + if (out_llama_cpp_return_code == nullptr) { + return LLAMA_RS_PROCESS_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG; } + *out_llama_cpp_return_code = 0; if (ctx == nullptr) { - return LLAMA_RS_DECODE_NULL_CTX_ARG; + return LLAMA_RS_PROCESS_NULL_CTX_ARG; } if (batch == nullptr) { - return LLAMA_RS_DECODE_NULL_BATCH_ARG; - } - if (out_error == nullptr) { - return LLAMA_RS_DECODE_NULL_OUT_ERROR_ARG; + return LLAMA_RS_PROCESS_NULL_BATCH_ARG; } try { - int32_t const result = llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch); + int32_t const result = llama_process(ctx, process_type, batch); if (result != 0) { - if (out_llama_cpp_return_code != nullptr) { - *out_llama_cpp_return_code = result; - } - if (result == -2) { - return LLAMA_RS_DECODE_OUT_OF_MEMORY; - } - if (result == -3) { - return LLAMA_RS_DECODE_COMPUTE_FAILED; - } - return LLAMA_RS_DECODE_LLAMA_CPP_RETURNED_NONZERO_CODE; + *out_llama_cpp_return_code = result; + return LLAMA_RS_PROCESS_LLAMA_CPP_RETURNED_NONZERO_CODE; } - return LLAMA_RS_DECODE_OK; + return LLAMA_RS_PROCESS_OK; } catch (...) { return llama_rs_report_current_exception( out_error, - LLAMA_RS_DECODE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_DECODE_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_DECODE_ERROR_STRING_ALLOCATION_FAILED); + LLAMA_RS_PROCESS_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_PROCESS_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_PROCESS_ERROR_STRING_ALLOCATION_FAILED); } } diff --git a/llama-cpp-bindings-sys/wrapper_common.h b/llama-cpp-bindings-sys/wrapper_common.h index a65763633..0c09463d3 100644 --- a/llama-cpp-bindings-sys/wrapper_common.h +++ b/llama-cpp-bindings-sys/wrapper_common.h @@ -126,26 +126,6 @@ llama_rs_memory_seq_pos_max_status llama_rs_memory_seq_pos_max( llama_pos * out_position, char ** out_error); -typedef enum llama_rs_encode_status { - LLAMA_RS_ENCODE_OK = 0, - LLAMA_RS_ENCODE_NULL_CTX_ARG, - LLAMA_RS_ENCODE_NULL_BATCH_ARG, - LLAMA_RS_ENCODE_NULL_MODEL, - LLAMA_RS_ENCODE_MODEL_HAS_NO_ENCODER, - LLAMA_RS_ENCODE_LLAMA_CPP_RETURNED_NONZERO_CODE, - LLAMA_RS_ENCODE_OUT_OF_MEMORY, - LLAMA_RS_ENCODE_COMPUTE_FAILED, - LLAMA_RS_ENCODE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_ENCODE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_ENCODE_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_encode_status; - -llama_rs_encode_status llama_rs_encode( - struct llama_context * ctx, - struct llama_batch_ext * batch, - int32_t * out_llama_cpp_return_code, - char ** out_error); - typedef enum llama_rs_memory_seq_add_status { LLAMA_RS_MEMORY_SEQ_ADD_OK = 0, LLAMA_RS_MEMORY_SEQ_ADD_NULL_CTX_ARG, @@ -218,21 +198,21 @@ llama_rs_new_context_with_model_status llama_rs_new_context_with_model( struct llama_context ** out_ctx, char ** out_error); -typedef enum llama_rs_decode_status { - LLAMA_RS_DECODE_OK = 0, - LLAMA_RS_DECODE_NULL_CTX_ARG, - LLAMA_RS_DECODE_NULL_BATCH_ARG, - LLAMA_RS_DECODE_NULL_OUT_ERROR_ARG, - LLAMA_RS_DECODE_LLAMA_CPP_RETURNED_NONZERO_CODE, - LLAMA_RS_DECODE_OUT_OF_MEMORY, - LLAMA_RS_DECODE_COMPUTE_FAILED, - LLAMA_RS_DECODE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_DECODE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_DECODE_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_decode_status; - -llama_rs_decode_status llama_rs_decode( +typedef enum llama_rs_process_status { + LLAMA_RS_PROCESS_OK = 0, + LLAMA_RS_PROCESS_NULL_CTX_ARG, + LLAMA_RS_PROCESS_NULL_BATCH_ARG, + LLAMA_RS_PROCESS_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG, + LLAMA_RS_PROCESS_NULL_OUT_ERROR_ARG, + LLAMA_RS_PROCESS_LLAMA_CPP_RETURNED_NONZERO_CODE, + LLAMA_RS_PROCESS_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_PROCESS_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_PROCESS_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_process_status; + +llama_rs_process_status llama_rs_process( struct llama_context * ctx, + enum llama_process_type process_type, struct llama_batch_ext * batch, int32_t * out_llama_cpp_return_code, char ** out_error); diff --git a/llama-cpp-bindings-tests/tests/encoder_requirement.rs b/llama-cpp-bindings-tests/tests/encoder_requirement.rs index db62a094c..7ea864c00 100644 --- a/llama-cpp-bindings-tests/tests/encoder_requirement.rs +++ b/llama-cpp-bindings-tests/tests/encoder_requirement.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use llama_cpp_bindings::EncodeError; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; @@ -29,7 +30,7 @@ fn encode_on_non_encoder_model_returns_error(fixture: &LlamaFixture<'_>) -> Resu let result = context.encode(&batch); - assert!(result.is_err()); + assert_eq!(result, Err(EncodeError::ModelHasNoEncoder)); Ok(()) } @@ -57,7 +58,7 @@ fn encode_succeeds_with_encoder_model(fixture: &LlamaFixture<'_>) -> Result<()> let result = context.encode(&batch); - assert!(result.is_ok()); + assert_eq!(result, Ok(())); Ok(()) } diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index ac27e7b9c..42eefb654 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -18,6 +18,7 @@ use crate::context::params::LlamaContextParams; use crate::context::params::LlamaPoolingType; use crate::context::sequence_output_capacity::SequenceOutputCapacity; use crate::error::batch_transfer_error::BatchTransferError; +use crate::error::process_failure::ProcessFailure; use crate::llama_backend::LlamaBackend; use crate::llama_batch::LlamaBatch; use crate::model::LlamaModel; @@ -289,123 +290,66 @@ fn batch_ext_add_entry_status_to_result( } } -fn decode_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_decode_status, +fn process_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_process_status, out_llama_cpp_return_code: i32, out_error: *mut c_char, -) -> Result<(), DecodeError> { - match status { - llama_cpp_bindings_sys::LLAMA_RS_DECODE_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_DECODE_LLAMA_CPP_RETURNED_NONZERO_CODE => { - let code = - NonZeroI32::new(out_llama_cpp_return_code).ok_or(crate::FfiContractError { - operation: "llama_rs_decode", - detail: "nonzero llama.cpp return status contained zero", - })?; - Err(DecodeError::from(code)) - } - llama_cpp_bindings_sys::LLAMA_RS_DECODE_OUT_OF_MEMORY => { - Err(DecodeError::DecodeOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_DECODE_COMPUTE_FAILED => Err(DecodeError::ComputeFailed), - llama_cpp_bindings_sys::LLAMA_RS_DECODE_ERROR_STRING_ALLOCATION_FAILED => { - Err(DecodeError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_DECODE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(DecodeError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_DECODE_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - out_error, - "llama_rs_decode", - "reported a thrown C++ exception without an error message", - ) - }?; - Err(DecodeError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_DECODE_NULL_CTX_ARG => Err(crate::FfiContractError { - operation: "llama_rs_decode", - detail: "was given a null ctx argument", - } - .into()), - llama_cpp_bindings_sys::LLAMA_RS_DECODE_NULL_BATCH_ARG => Err(crate::FfiContractError { - operation: "llama_rs_decode", - detail: "was given a null batch argument", - } - .into()), - llama_cpp_bindings_sys::LLAMA_RS_DECODE_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_decode", - detail: "was given a null out_error argument", - } - .into()) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_decode", - code: i64::from(other), - } - .into()), - } -} +) -> Result<(), TProcessError> +where + TProcessError: From + From, +{ + const OPERATION: &str = "llama_rs_process"; + + let contract_violation = |detail: &'static str| { + ProcessFailure::from(crate::FfiContractError { + operation: OPERATION, + detail, + }) + }; -fn encode_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_encode_status, - out_llama_cpp_return_code: i32, - out_error: *mut c_char, -) -> Result<(), EncodeError> { match status { - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_MODEL_HAS_NO_ENCODER => { - Err(EncodeError::ModelHasNoEncoder) - } - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_LLAMA_CPP_RETURNED_NONZERO_CODE => { - let code = - NonZeroI32::new(out_llama_cpp_return_code).ok_or(crate::FfiContractError { - operation: "llama_rs_encode", - detail: "nonzero llama.cpp return status contained zero", - })?; - Err(EncodeError::from(code)) - } - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_OUT_OF_MEMORY => { - Err(EncodeError::EncodeOutOfMemory) + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_OK => Ok(()), + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_LLAMA_CPP_RETURNED_NONZERO_CODE => { + let code = NonZeroI32::new(out_llama_cpp_return_code).ok_or_else(|| { + contract_violation("nonzero llama.cpp return status contained zero") + })?; + + Err(TProcessError::from(code)) } - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_COMPUTE_FAILED => Err(EncodeError::ComputeFailed), - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_ERROR_STRING_ALLOCATION_FAILED => { - Err(EncodeError::NotEnoughMemory) + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_ERROR_STRING_ALLOCATION_FAILED => { + Err(ProcessFailure::NotEnoughMemory.into()) } - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(EncodeError::LlamaCppOutOfMemory) + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_LLAMA_CPP_OUT_OF_MEMORY => { + Err(ProcessFailure::LlamaCppOutOfMemory.into()) } - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_LLAMA_CPP_THREW_CXX_EXCEPTION => { + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_LLAMA_CPP_THREW_CXX_EXCEPTION => { let message = unsafe { read_and_free_cpp_string( out_error, - "llama_rs_encode", + OPERATION, "reported a thrown C++ exception without an error message", ) - }?; - Err(EncodeError::Reported { message }) + } + .map_err(ProcessFailure::from)?; + + Err(ProcessFailure::Reported { message }.into()) } - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_NULL_CTX_ARG => Err(crate::FfiContractError { - operation: "llama_rs_encode", - detail: "was given a null ctx argument", + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_NULL_CTX_ARG => { + Err(contract_violation("was given a null ctx argument").into()) } - .into()), - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_NULL_BATCH_ARG => Err(crate::FfiContractError { - operation: "llama_rs_encode", - detail: "was given a null batch argument", + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_NULL_BATCH_ARG => { + Err(contract_violation("was given a null batch argument").into()) } - .into()), - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_NULL_MODEL => Err(crate::FfiContractError { - operation: "llama_rs_encode", - detail: "was given a null model argument", + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG => { + Err(contract_violation("was given a null out_llama_cpp_return_code argument").into()) } - .into()), - other => Err(crate::FfiStatusError { - operation: "llama_rs_encode", - code: i64::from(other), + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_NULL_OUT_ERROR_ARG => { + Err(contract_violation("was given a null out_error argument").into()) } + other => Err(ProcessFailure::from(crate::FfiStatusError { + operation: OPERATION, + code: i64::from(other), + }) .into()), } } @@ -511,9 +455,7 @@ impl<'model> LlamaContext<'model> { model_causal_attention: unsafe { llama_cpp_bindings_sys::llama_rs_model_causal_attn(model.model.as_ptr()) }, - model_has_encoder: unsafe { - llama_cpp_bindings_sys::llama_model_has_encoder(model.model.as_ptr()) - }, + model_has_encoder: model.has_encoder(), model_n_ctx_train: unsafe { llama_cpp_bindings_sys::llama_model_n_ctx_train(model.model.as_ptr()) } @@ -654,31 +596,29 @@ impl<'model> LlamaContext<'model> { Ok(()) } - /// # Errors - /// - /// - `DecodeError` if the decoding failed. - pub fn decode(&mut self, batch: &LlamaBatch) -> Result<(), DecodeError> { - if self.decodes_batches_in_one_micro_batch() && self.exceeds_micro_batch(batch) { - return Err(DecodeError::BatchExceedsMicroBatch { - n_tokens: batch.n_tokens(), - n_ubatch: self.n_ubatch(), - }); - } - + fn process( + &mut self, + process_type: llama_cpp_bindings_sys::llama_process_type, + batch: &LlamaBatch, + ) -> Result<(), TProcessError> + where + TProcessError: From + From + From, + { self.transfer_batch(batch)?; let mut out_llama_cpp_return_code: i32 = 0; let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { - llama_cpp_bindings_sys::llama_rs_decode( + llama_cpp_bindings_sys::llama_rs_process( self.context.as_ptr(), + process_type, self.batch_ext.as_ptr(), &raw mut out_llama_cpp_return_code, &raw mut out_error, ) }; - decode_status_to_result(status, out_llama_cpp_return_code, out_error)?; + process_status_to_result::(status, out_llama_cpp_return_code, out_error)?; self.record_decoded_rows(DecodedOutputRows::BatchIndices( batch.output_token_indices().to_vec(), )); @@ -688,7 +628,22 @@ impl<'model> LlamaContext<'model> { /// # Errors /// - /// - `EncodeError` if the encoding failed. + /// - `DecodeError` if the decoding failed. + pub fn decode(&mut self, batch: &LlamaBatch) -> Result<(), DecodeError> { + if self.decodes_batches_in_one_micro_batch() && self.exceeds_micro_batch(batch) { + return Err(DecodeError::BatchExceedsMicroBatch { + n_tokens: batch.n_tokens(), + n_ubatch: self.n_ubatch(), + }); + } + + self.process(llama_cpp_bindings_sys::LLAMA_PROCESS_TYPE_DECODE, batch) + } + + /// # Errors + /// + /// - `EncodeError` if the encoding failed, or [`EncodeError::ModelHasNoEncoder`] when the + /// model has no encoder to run. pub fn encode(&mut self, batch: &LlamaBatch) -> Result<(), EncodeError> { if self.exceeds_micro_batch(batch) { return Err(EncodeError::BatchExceedsMicroBatch { @@ -697,25 +652,11 @@ impl<'model> LlamaContext<'model> { }); } - self.transfer_batch(batch)?; - - let mut out_llama_cpp_return_code: i32 = 0; - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_encode( - self.context.as_ptr(), - self.batch_ext.as_ptr(), - &raw mut out_llama_cpp_return_code, - &raw mut out_error, - ) - }; - encode_status_to_result(status, out_llama_cpp_return_code, out_error)?; - - self.record_decoded_rows(DecodedOutputRows::BatchIndices( - batch.output_token_indices().to_vec(), - )); + if !self.model.has_encoder() { + return Err(EncodeError::ModelHasNoEncoder); + } - Ok(()) + self.process(llama_cpp_bindings_sys::LLAMA_PROCESS_TYPE_ENCODE, batch) } #[must_use] @@ -1621,384 +1562,114 @@ mod batch_ext_add_entry_status_to_result_tests { } #[cfg(test)] -mod decode_status_to_result_tests { +mod process_status_to_result_tests { use std::ptr; - use super::decode_status_to_result; + use super::process_status_to_result; use crate::DecodeError; + use crate::EncodeError; + use crate::error::process_failure::ProcessFailure; - #[test] - fn decode_nonzero_code_maps_from_code() { - let result = decode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DECODE_LLAMA_CPP_RETURNED_NONZERO_CODE, - 1, - ptr::null_mut(), - ); - - assert_eq!(result, Err(DecodeError::NoKvCacheSlot)); - } - - #[test] - fn decode_out_of_memory_maps_decode_out_of_memory() { - let result = decode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DECODE_OUT_OF_MEMORY, - 0, - ptr::null_mut(), - ); - - assert_eq!(result, Err(DecodeError::DecodeOutOfMemory)); - } - - #[test] - fn decode_compute_failed_maps_compute_failed() { - let result = decode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DECODE_COMPUTE_FAILED, - 0, - ptr::null_mut(), - ); - - assert_eq!(result, Err(DecodeError::ComputeFailed)); - } - - #[test] - fn decode_allocation_failed_maps_not_enough_memory() { - let result = decode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DECODE_ERROR_STRING_ALLOCATION_FAILED, - 0, - ptr::null_mut(), - ); - - assert_eq!(result, Err(DecodeError::NotEnoughMemory)); - } - - #[test] - fn decode_cxx_exception_reports_its_message() { - let out_error = - unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"decode threw".as_ptr()) }; - let result = decode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DECODE_LLAMA_CPP_THREW_CXX_EXCEPTION, - 0, - out_error, - ); - - assert_eq!( - result, - Err(DecodeError::Reported { - message: "decode threw".to_owned() - }) - ); - } - - #[test] - fn decode_cxx_exception_without_a_message_is_a_contract_error() { - let result = decode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DECODE_LLAMA_CPP_THREW_CXX_EXCEPTION, - 0, - ptr::null_mut(), - ); - - assert_eq!( - result, - Err(crate::FfiContractError { - operation: "llama_rs_decode", - detail: "reported a thrown C++ exception without an error message", - } - .into()) - ); - } - - #[test] - fn decode_nonzero_status_with_zero_code_is_contract_error() { - let result = decode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DECODE_LLAMA_CPP_RETURNED_NONZERO_CODE, - 0, - ptr::null_mut(), - ); + const OPERATION: &str = "llama_rs_process"; - assert_eq!( - result, - Err(DecodeError::FfiContract(crate::FfiContractError { - operation: "llama_rs_decode", - detail: "nonzero llama.cpp return status contained zero", - })) - ); + fn contract_violation(detail: &'static str) -> DecodeError { + ProcessFailure::from(crate::FfiContractError { + operation: OPERATION, + detail, + }) + .into() } #[test] - fn decode_unknown_status_is_preserved() { - let result = decode_status_to_result(255, 0, ptr::null_mut()); + fn every_status_without_a_message_maps_to_its_error() { + let statuses = [ + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_OK, + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_LLAMA_CPP_RETURNED_NONZERO_CODE, + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_ERROR_STRING_ALLOCATION_FAILED, + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_LLAMA_CPP_OUT_OF_MEMORY, + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_LLAMA_CPP_THREW_CXX_EXCEPTION, + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_NULL_CTX_ARG, + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_NULL_BATCH_ARG, + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG, + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_NULL_OUT_ERROR_ARG, + 255, + ]; assert_eq!( - result, - Err(DecodeError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_decode", - code: 255, - })) - ); - } - - #[test] - fn decode_status_to_result_maps_null_ctx_arg() { - let outcome = decode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DECODE_NULL_CTX_ARG, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_decode", - detail: "was given a null ctx argument", - } - .into() - ) + statuses.map(|status| process_status_to_result::( + status, + 0, + ptr::null_mut() + )), + [ + Ok(()), + Err(contract_violation( + "nonzero llama.cpp return status contained zero" + )), + Err(ProcessFailure::NotEnoughMemory.into()), + Err(ProcessFailure::LlamaCppOutOfMemory.into()), + Err(contract_violation( + "reported a thrown C++ exception without an error message" + )), + Err(contract_violation("was given a null ctx argument")), + Err(contract_violation("was given a null batch argument")), + Err(contract_violation( + "was given a null out_llama_cpp_return_code argument" + )), + Err(contract_violation("was given a null out_error argument")), + Err(ProcessFailure::from(crate::FfiStatusError { + operation: OPERATION, + code: 255, + }) + .into()), + ] ); } #[test] - fn decode_status_to_result_maps_null_batch_arg() { - let outcome = decode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DECODE_NULL_BATCH_ARG, - 0, - ptr::null_mut(), - ); + fn a_nonzero_llama_cpp_code_is_read_as_the_error_of_the_process_type() { assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_decode", - detail: "was given a null batch argument", - } - .into() - ) - ); - } - - #[test] - fn decode_status_to_result_maps_null_out_error_arg() { - let outcome = decode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DECODE_NULL_OUT_ERROR_ARG, - 0, - ptr::null_mut(), + [ + process_status_to_result::( + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_LLAMA_CPP_RETURNED_NONZERO_CODE, + 2, + ptr::null_mut() + ), + process_status_to_result::( + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_LLAMA_CPP_RETURNED_NONZERO_CODE, + -3, + ptr::null_mut() + ), + ], + [Err(DecodeError::Aborted), Err(DecodeError::ComputeFailed)] ); assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_decode", - detail: "was given a null out_error argument", - } - .into() - ) - ); - } - - #[test] - fn decode_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = decode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DECODE_LLAMA_CPP_OUT_OF_MEMORY, - 0, - ptr::null_mut(), - ); - assert_eq!(outcome.err(), Some(DecodeError::LlamaCppOutOfMemory)); - } -} - -#[cfg(test)] -mod encode_status_to_result_tests { - use std::ptr; - - use super::encode_status_to_result; - use crate::EncodeError; - - #[test] - fn encode_model_has_no_encoder_maps_model_has_no_encoder() { - let result = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_MODEL_HAS_NO_ENCODER, - 0, - ptr::null_mut(), - ); - - assert_eq!(result, Err(EncodeError::ModelHasNoEncoder)); - } - - #[test] - fn encode_nonzero_code_maps_from_code() { - let result = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_LLAMA_CPP_RETURNED_NONZERO_CODE, - 1, - ptr::null_mut(), - ); - - assert_eq!(result, Err(EncodeError::NoKvCacheSlot)); - } - - #[test] - fn encode_out_of_memory_maps_encode_out_of_memory() { - let result = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_OUT_OF_MEMORY, - 0, - ptr::null_mut(), - ); - - assert_eq!(result, Err(EncodeError::EncodeOutOfMemory)); - } - - #[test] - fn encode_compute_failed_maps_compute_failed() { - let result = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_COMPUTE_FAILED, - 0, - ptr::null_mut(), - ); - - assert_eq!(result, Err(EncodeError::ComputeFailed)); - } - - #[test] - fn encode_allocation_failed_maps_not_enough_memory() { - let result = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_ERROR_STRING_ALLOCATION_FAILED, - 0, - ptr::null_mut(), + process_status_to_result::( + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_LLAMA_CPP_RETURNED_NONZERO_CODE, + -2, + ptr::null_mut() + ), + Err(EncodeError::EncodeOutOfMemory) ); - - assert_eq!(result, Err(EncodeError::NotEnoughMemory)); } #[test] - fn encode_cxx_exception_reports_its_message() { + fn a_thrown_exception_reports_its_message() { let out_error = - unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"encode threw".as_ptr()) }; - let result = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_LLAMA_CPP_THREW_CXX_EXCEPTION, - 0, - out_error, - ); + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"process threw".as_ptr()) }; assert_eq!( - result, - Err(EncodeError::Reported { - message: "encode threw".to_owned() - }) - ); - } - - #[test] - fn encode_cxx_exception_without_a_message_is_a_contract_error() { - let result = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_LLAMA_CPP_THREW_CXX_EXCEPTION, - 0, - ptr::null_mut(), - ); - - assert_eq!( - result, - Err(crate::FfiContractError { - operation: "llama_rs_encode", - detail: "reported a thrown C++ exception without an error message", + process_status_to_result::( + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + out_error, + ), + Err(ProcessFailure::Reported { + message: "process threw".to_owned() } .into()) ); } - - #[test] - fn encode_nonzero_status_with_zero_code_is_contract_error() { - let result = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_LLAMA_CPP_RETURNED_NONZERO_CODE, - 0, - ptr::null_mut(), - ); - - assert_eq!( - result, - Err(EncodeError::FfiContract(crate::FfiContractError { - operation: "llama_rs_encode", - detail: "nonzero llama.cpp return status contained zero", - })) - ); - } - - #[test] - fn encode_unknown_status_is_preserved() { - let result = encode_status_to_result(255, 0, ptr::null_mut()); - - assert_eq!( - result, - Err(EncodeError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_encode", - code: 255, - })) - ); - } - - #[test] - fn encode_status_to_result_maps_null_ctx_arg() { - let outcome = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_NULL_CTX_ARG, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_encode", - detail: "was given a null ctx argument", - } - .into() - ) - ); - } - - #[test] - fn encode_status_to_result_maps_null_batch_arg() { - let outcome = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_NULL_BATCH_ARG, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_encode", - detail: "was given a null batch argument", - } - .into() - ) - ); - } - - #[test] - fn encode_status_to_result_maps_null_model() { - let outcome = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_NULL_MODEL, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_encode", - detail: "was given a null model argument", - } - .into() - ) - ); - } - - #[test] - fn encode_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_LLAMA_CPP_OUT_OF_MEMORY, - 0, - ptr::null_mut(), - ); - assert_eq!(outcome.err(), Some(EncodeError::LlamaCppOutOfMemory)); - } } #[cfg(test)] diff --git a/llama-cpp-bindings/src/error.rs b/llama-cpp-bindings/src/error.rs index 374b80e73..2c95040a0 100644 --- a/llama-cpp-bindings/src/error.rs +++ b/llama-cpp-bindings/src/error.rs @@ -30,6 +30,7 @@ pub mod model_params_error; pub mod new_llama_chat_message_error; pub mod paired_quote_failure; pub mod parse_chat_message_error; +pub mod process_failure; pub mod sample_error; pub mod sampler_accept_error; pub mod sampler_apply_error; @@ -77,6 +78,7 @@ pub use model_params_error::ModelParamsError; pub use new_llama_chat_message_error::NewLlamaChatMessageError; pub use paired_quote_failure::PairedQuoteFailure; pub use parse_chat_message_error::ParseChatMessageError; +pub use process_failure::ProcessFailure; pub use sample_error::SampleError; pub use sampler_accept_error::SamplerAcceptError; pub use sampler_apply_error::SamplerApplyError; diff --git a/llama-cpp-bindings/src/error/decode_error.rs b/llama-cpp-bindings/src/error/decode_error.rs index ad41b9710..272bc5702 100644 --- a/llama-cpp-bindings/src/error/decode_error.rs +++ b/llama-cpp-bindings/src/error/decode_error.rs @@ -2,13 +2,12 @@ use std::num::NonZeroI32; use std::os::raw::c_int; use crate::error::batch_transfer_error::BatchTransferError; +use crate::error::process_failure::ProcessFailure; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum DecodeError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), - #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + Process(#[from] ProcessFailure), #[error("no KV cache slot was available")] NoKvCacheSlot, #[error("decode aborted by callback")] @@ -21,16 +20,10 @@ pub enum DecodeError { ComputeFailed, #[error("decode returned an unknown status code: {code}")] UnknownStatus { code: c_int }, - #[error("not enough memory")] - NotEnoughMemory, - #[error("the llama.cpp library ran out of memory")] - LlamaCppOutOfMemory, #[error("decode batch of {n_tokens} tokens must fit a single micro batch of {n_ubatch} tokens")] BatchExceedsMicroBatch { n_tokens: i32, n_ubatch: u32 }, #[error("the batch could not be handed to llama.cpp: {0}")] BatchTransfer(#[from] BatchTransferError), - #[error("{message}")] - Reported { message: String }, } impl From for DecodeError { @@ -39,6 +32,8 @@ impl From for DecodeError { 1 => Self::NoKvCacheSlot, 2 => Self::Aborted, -1 => Self::BatchInvalid, + -2 => Self::DecodeOutOfMemory, + -3 => Self::ComputeFailed, error_code => Self::UnknownStatus { code: error_code }, } } @@ -71,6 +66,14 @@ mod tests { assert_eq!(error, DecodeError::BatchInvalid); } + #[test] + fn allocation_and_compute_failures_map_from_codes_negative_two_and_three() { + assert_eq!( + [-2, -3].map(|code| DecodeError::from(NonZeroI32::new(code).expect("non-zero code"))), + [DecodeError::DecodeOutOfMemory, DecodeError::ComputeFailed] + ); + } + #[test] fn unrecognized_code_falls_through_to_unknown_status() { let error = DecodeError::from(NonZeroI32::new(42).expect("42 is non-zero")); diff --git a/llama-cpp-bindings/src/error/encode_error.rs b/llama-cpp-bindings/src/error/encode_error.rs index 4b0d605c6..72113234b 100644 --- a/llama-cpp-bindings/src/error/encode_error.rs +++ b/llama-cpp-bindings/src/error/encode_error.rs @@ -2,13 +2,12 @@ use std::num::NonZeroI32; use std::os::raw::c_int; use crate::error::batch_transfer_error::BatchTransferError; +use crate::error::process_failure::ProcessFailure; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum EncodeError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), - #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + Process(#[from] ProcessFailure), #[error("model has no encoder")] ModelHasNoEncoder, #[error("no KV cache slot was available")] @@ -21,16 +20,10 @@ pub enum EncodeError { ComputeFailed, #[error("encode returned an unknown status code: {code}")] UnknownStatus { code: c_int }, - #[error("not enough memory")] - NotEnoughMemory, - #[error("the llama.cpp library ran out of memory")] - LlamaCppOutOfMemory, #[error("encode batch of {n_tokens} tokens must fit a single micro batch of {n_ubatch} tokens")] BatchExceedsMicroBatch { n_tokens: i32, n_ubatch: u32 }, #[error("the batch could not be handed to llama.cpp: {0}")] BatchTransfer(#[from] BatchTransferError), - #[error("{message}")] - Reported { message: String }, } impl From for EncodeError { @@ -38,6 +31,8 @@ impl From for EncodeError { match value.get() { 1 => Self::NoKvCacheSlot, -1 => Self::BatchInvalid, + -2 => Self::EncodeOutOfMemory, + -3 => Self::ComputeFailed, error_code => Self::UnknownStatus { code: error_code }, } } @@ -63,6 +58,14 @@ mod tests { assert_eq!(error, EncodeError::BatchInvalid); } + #[test] + fn allocation_and_compute_failures_map_from_codes_negative_two_and_three() { + assert_eq!( + [-2, -3].map(|code| EncodeError::from(NonZeroI32::new(code).expect("non-zero code"))), + [EncodeError::EncodeOutOfMemory, EncodeError::ComputeFailed] + ); + } + #[test] fn unrecognized_code_falls_through_to_unknown_status() { let error = EncodeError::from(NonZeroI32::new(99).expect("99 is non-zero")); diff --git a/llama-cpp-bindings/src/error/process_failure.rs b/llama-cpp-bindings/src/error/process_failure.rs new file mode 100644 index 000000000..73e5ba4f1 --- /dev/null +++ b/llama-cpp-bindings/src/error/process_failure.rs @@ -0,0 +1,14 @@ +/// How `llama_rs_process` failed before llama.cpp returned a code of its own. +#[derive(Debug, Eq, PartialEq, thiserror::Error)] +pub enum ProcessFailure { + #[error(transparent)] + FfiStatus(#[from] crate::FfiStatusError), + #[error(transparent)] + FfiContract(#[from] crate::FfiContractError), + #[error("not enough memory")] + NotEnoughMemory, + #[error("the llama.cpp library ran out of memory")] + LlamaCppOutOfMemory, + #[error("{message}")] + Reported { message: String }, +} diff --git a/llama-cpp-bindings/src/lib.rs b/llama-cpp-bindings/src/lib.rs index 6ad67b92d..a9d3e9c9d 100644 --- a/llama-cpp-bindings/src/lib.rs +++ b/llama-cpp-bindings/src/lib.rs @@ -88,8 +88,8 @@ pub use error::{ KvCacheSeqAddError, KvCacheSeqDivError, KvCacheSeqPosMaxError, LlamaContextLoadError, LlamaCppError, LlamaLoraAdapterInitError, LlamaLoraAdaptersError, LlamaModelLoadError, LogitsError, MetaValError, ModelParamsError, NewLlamaChatMessageError, ParseChatMessageError, - Result, SampleError, SamplerAcceptError, SamplingError, StringToTokenError, TokenAttrError, - TokenOutsideVocabulary, TokenSamplingError, TokenToStringError, + ProcessFailure, Result, SampleError, SamplerAcceptError, SamplingError, StringToTokenError, + TokenAttrError, TokenOutsideVocabulary, TokenSamplingError, TokenToStringError, }; pub use bare_json_tool_calls::BareJsonToolCalls; diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index 1a1c4c915..3582d6938 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -470,6 +470,11 @@ impl LlamaModel { unsafe { llama_cpp_bindings_sys::llama_model_is_hybrid(self.model.as_ptr()) } } + #[must_use] + pub fn has_encoder(&self) -> bool { + unsafe { llama_cpp_bindings_sys::llama_model_has_encoder(self.model.as_ptr()) } + } + /// # Errors /// Returns an error if the key is not found or the value is not valid UTF-8. pub fn meta_val_str(&self, key: &str) -> Result { From 7e25121d243388fc6812d4308ffbd7025168815d Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Fri, 30 Oct 2026 23:47:41 +0100 Subject: [PATCH 20/82] Name the vision and logging tests after what they verify, move shared test helpers to the support crate and drop debug prints from tests --- .../context_after_decoding_text.rs | 23 +++++++++++----- llama-cpp-bindings-tests/src/lib.rs | 2 ++ .../tokenize_prompt_with_synthetic_image.rs} | 21 ++++++++++++--- .../tests/backend_initialization.rs | 2 +- .../tests/completion_usage_accounting.rs | 27 ------------------- .../tests/embedding_availability.rs | 15 ++++------- .../tests/embedding_generation.rs | 14 ---------- .../tests/embedding_row_width.rs | 8 +++--- ...ference.rs => image_question_answering.rs} | 16 +++-------- .../json_schema_constrained_generation.rs | 6 ----- .../tests/logits_availability.rs | 26 +++++++----------- llama-cpp-bindings-tests/tests/main.rs | 4 +-- .../tests/multimodal_chunk_evaluation.rs | 14 +++++----- .../tests/multimodal_chunk_tokenization.rs | 16 +++++------ .../tests/multimodal_fusion.rs | 4 ++- llama-cpp-bindings-tests/tests/reranking.rs | 15 ----------- 16 files changed, 76 insertions(+), 137 deletions(-) rename llama-cpp-bindings-tests/{tests => src}/context_after_decoding_text.rs (63%) rename llama-cpp-bindings-tests/{tests/tokenize_synthetic.rs => src/tokenize_prompt_with_synthetic_image.rs} (51%) rename llama-cpp-bindings-tests/tests/{multimodal_vision_inference.rs => image_question_answering.rs} (91%) diff --git a/llama-cpp-bindings-tests/tests/context_after_decoding_text.rs b/llama-cpp-bindings-tests/src/context_after_decoding_text.rs similarity index 63% rename from llama-cpp-bindings-tests/tests/context_after_decoding_text.rs rename to llama-cpp-bindings-tests/src/context_after_decoding_text.rs index ea559dfee..64c815a12 100644 --- a/llama-cpp-bindings-tests/tests/context_after_decoding_text.rs +++ b/llama-cpp-bindings-tests/src/context_after_decoding_text.rs @@ -1,4 +1,3 @@ -use anyhow::Result; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::context::params::LlamaPoolingType; @@ -9,24 +8,34 @@ use llama_cpp_test_harness::LlamaFixture; const TEXT: &str = "The parcel arrived on the last working day of the week."; +/// A context with `pooling_type` that has decoded one sentence as sequence 0. +/// +/// # Panics +/// +/// Panics when the fixture's context cannot be created or the sentence cannot be decoded in it. +#[must_use] pub fn context_after_decoding_text<'fixture>( fixture: &'fixture LlamaFixture<'_>, pooling_type: LlamaPoolingType, -) -> Result> { +) -> LlamaContext<'fixture> { let mut context = LlamaContext::from_model( fixture.model, fixture.backend, (*fixture.context_params) .into_llama_context_params() .with_pooling_type(pooling_type), - )?; + ) + .expect("the fixture's context must be creatable"); let tokens = fixture .model - .str_to_token(TEXT, AddBos::Always, ParseSpecialTokens::Never)?; + .str_to_token(TEXT, AddBos::Always, ParseSpecialTokens::Never) + .expect("the sentence must tokenize"); let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; + batch + .add_sequence(&tokens, 0, false) + .expect("the sentence must fit one batch"); + context.decode(&batch).expect("the sentence must decode"); - Ok(context) + context } diff --git a/llama-cpp-bindings-tests/src/lib.rs b/llama-cpp-bindings-tests/src/lib.rs index 727c6acef..fc21c709a 100644 --- a/llama-cpp-bindings-tests/src/lib.rs +++ b/llama-cpp-bindings-tests/src/lib.rs @@ -5,6 +5,7 @@ pub mod chunk_token_breakdown; pub mod classify_sample_loop; pub mod classify_sample_loop_outcome; pub mod closed_reasoning_chat_session; +pub mod context_after_decoding_text; pub mod fixtures_dir; pub mod get_weather_tools_json; pub mod ingested_generation; @@ -16,5 +17,6 @@ pub mod prime_kv_cache; pub mod prime_kv_cache_with; pub mod prompt_question; pub mod recognized_answer; +pub mod tokenize_prompt_with_synthetic_image; pub mod user_turn_chat_session; pub mod weather_chat_session; diff --git a/llama-cpp-bindings-tests/tests/tokenize_synthetic.rs b/llama-cpp-bindings-tests/src/tokenize_prompt_with_synthetic_image.rs similarity index 51% rename from llama-cpp-bindings-tests/tests/tokenize_synthetic.rs rename to llama-cpp-bindings-tests/src/tokenize_prompt_with_synthetic_image.rs index 8b8b6a087..7eb7a3aed 100644 --- a/llama-cpp-bindings-tests/tests/tokenize_synthetic.rs +++ b/llama-cpp-bindings-tests/src/tokenize_prompt_with_synthetic_image.rs @@ -1,19 +1,32 @@ -use anyhow::Result; use llama_cpp_bindings::mtmd::MtmdBitmap; use llama_cpp_bindings::mtmd::MtmdInputChunks; use llama_cpp_bindings::mtmd::MtmdInputText; use llama_cpp_test_harness::LlamaFixture; -pub fn tokenize_synthetic(fixture: &LlamaFixture<'_>, prompt: &str) -> Result { +/// Tokenizes `prompt` with a uniformly grey 64x64 image as its media. +/// +/// # Panics +/// +/// Panics when the fixture declares no multimodal projector, or the image or prompt cannot be +/// tokenized. +#[must_use] +pub fn tokenize_prompt_with_synthetic_image( + fixture: &LlamaFixture<'_>, + prompt: &str, +) -> MtmdInputChunks { let mtmd_ctx = fixture .mtmd_context .expect("mmproj_file declared in attribute"); let image_data = vec![128u8; 64 * 64 * 3]; - let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; + let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data) + .expect("a 64x64 RGB image must make a bitmap"); let input_text = MtmdInputText { text: prompt.to_owned(), add_special: true, parse_special: true, }; - Ok(mtmd_ctx.tokenize(input_text, &[&bitmap])?) + + mtmd_ctx + .tokenize(input_text, &[&bitmap]) + .expect("the prompt and image must tokenize") } diff --git a/llama-cpp-bindings-tests/tests/backend_initialization.rs b/llama-cpp-bindings-tests/tests/backend_initialization.rs index c4fcc4e33..e1b030876 100644 --- a/llama-cpp-bindings-tests/tests/backend_initialization.rs +++ b/llama-cpp-bindings-tests/tests/backend_initialization.rs @@ -10,7 +10,7 @@ use llama_cpp_test_harness::llama_test; n_ubatch = 64, void_logs = true, )] -fn void_logs_suppresses_output(fixture: &LlamaFixture<'_>) { +fn a_model_loads_after_llama_cpp_logging_is_silenced(fixture: &LlamaFixture<'_>) { assert!( fixture.model.n_vocab() > 0, "model must load successfully even when void_logs has been called before init" diff --git a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs index f2c50c0f9..f644ccf34 100644 --- a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs +++ b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs @@ -1,6 +1,3 @@ -use std::io::Write; -use std::time::Duration; - use anyhow::Context as _; use anyhow::Result; use llama_cpp_bindings::BareJsonToolCalls; @@ -8,11 +5,9 @@ use llama_cpp_bindings::ChatAssistantTurn; use llama_cpp_bindings::ChatSession; use llama_cpp_bindings::ChatSessionOptions; use llama_cpp_bindings::ChatToolChoice; -use llama_cpp_bindings::SampledToken; use llama_cpp_bindings::SampledTokenClassifier; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; -use llama_cpp_bindings::ggml_time_us; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::AddBos; use llama_cpp_bindings::model::LlamaChatMessage; @@ -75,16 +70,6 @@ fn raw_prompt_completion_usage_matches_the_classified_tokens( .context(format!("failed to tokenize {prompt}"))?; let prompt_token_count = u64::try_from(tokens_list.len())?; - let mut decoder = encoding_rs::UTF_8.new_decoder(); - - for token in &tokens_list { - eprint!( - "{}", - model.token_to_piece(&SampledToken::Content(*token), &mut decoder, true, None)? - ); - } - std::io::stderr().flush()?; - let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens_list, 0, false)?; @@ -101,7 +86,6 @@ fn raw_prompt_completion_usage_matches_the_classified_tokens( let mut sampler = LlamaSampler::chain_simple([LlamaSampler::dist(1234)?, LlamaSampler::greedy()?])?; let initial_position = batch.n_tokens(); - let t_main_start = ggml_time_us(); let outcome = ClassifySampleLoop { classifier: &mut classifier, sampler: &mut sampler, @@ -111,18 +95,9 @@ fn raw_prompt_completion_usage_matches_the_classified_tokens( max_generated_tokens, } .run(); - let t_main_end = ggml_time_us(); - let duration = Duration::from_micros(u64::try_from(t_main_end - t_main_start)?); let total_observed = outcome.observed_content + outcome.observed_reasoning + outcome.observed_undeterminable; - let tokens_per_second = f64::from(u32::try_from(total_observed)?) / duration.as_secs_f64(); - - eprintln!( - "\ndecoded {total_observed} tokens in {:.2} s, speed {tokens_per_second:.2} t/s", - duration.as_secs_f64(), - ); - assert!( !outcome.generated.text().is_empty(), "model should generate at least one token" @@ -252,8 +227,6 @@ fn chat_completion_usage_matches_the_classified_tokens(fixture: &LlamaFixture<'_ } .run(); - println!(); - assert!( !outcome.generated.text().is_empty(), "model should generate at least one token" diff --git a/llama-cpp-bindings-tests/tests/embedding_availability.rs b/llama-cpp-bindings-tests/tests/embedding_availability.rs index 0bc533cfc..6639fe796 100644 --- a/llama-cpp-bindings-tests/tests/embedding_availability.rs +++ b/llama-cpp-bindings-tests/tests/embedding_availability.rs @@ -1,10 +1,9 @@ -use anyhow::Result; use llama_cpp_bindings::EmbeddingsError; use llama_cpp_bindings::context::params::LlamaPoolingType; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; -use crate::context_after_decoding_text::context_after_decoding_text; +use llama_cpp_bindings_tests::context_after_decoding_text::context_after_decoding_text; #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), @@ -15,15 +14,13 @@ use crate::context_after_decoding_text::context_after_decoding_text; n_ubatch = 512, embeddings = true, )] -fn sequence_embeddings_need_a_pooling_context(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = context_after_decoding_text(fixture, LlamaPoolingType::None)?; +fn sequence_embeddings_need_a_pooling_context(fixture: &LlamaFixture<'_>) { + let context = context_after_decoding_text(fixture, LlamaPoolingType::None); assert_eq!( context.embeddings_seq_ith(0), Err(EmbeddingsError::NonePoolType) ); - - Ok(()) } #[llama_test( @@ -35,8 +32,8 @@ fn sequence_embeddings_need_a_pooling_context(fixture: &LlamaFixture<'_>) -> Res n_ubatch = 512, embeddings = true, )] -fn token_embeddings_need_a_context_without_pooling(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean)?; +fn token_embeddings_need_a_context_without_pooling(fixture: &LlamaFixture<'_>) { + let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean); assert_eq!( context.embeddings_ith(0), @@ -44,6 +41,4 @@ fn token_embeddings_need_a_context_without_pooling(fixture: &LlamaFixture<'_>) - pooling_type: LlamaPoolingType::Mean, }) ); - - Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/embedding_generation.rs b/llama-cpp-bindings-tests/tests/embedding_generation.rs index 60f1b96af..c119b5463 100644 --- a/llama-cpp-bindings-tests/tests/embedding_generation.rs +++ b/llama-cpp-bindings-tests/tests/embedding_generation.rs @@ -1,12 +1,9 @@ -use std::time::Duration; - use anyhow::Context; use anyhow::Result; use llama_cpp_bindings::SampledTokenClassifier; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::context::params::LlamaPoolingType; -use llama_cpp_bindings::ggml_time_us; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::AddBos; use llama_cpp_bindings::model::ParseSpecialTokens; @@ -43,8 +40,6 @@ fn embedding_generation_produces_vectors(fixture: &LlamaFixture<'_>) -> Result<( let n_ctx = usize::try_from(ctx.n_ctx())?; assert!(tokens.len() <= n_ctx, "prompt exceeds context window size"); - let t_main_start = ggml_time_us(); - let mut classifier = SampledTokenClassifier::undeterminable(model); let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; @@ -64,15 +59,6 @@ fn embedding_generation_produces_vectors(fixture: &LlamaFixture<'_>) -> Result<( .with_context(|| "failed to get embeddings")?; let normalized = normalize(embedding); - let t_main_end = ggml_time_us(); - let duration = Duration::from_micros(u64::try_from(t_main_end - t_main_start)?); - - eprintln!( - "created embedding with {} dimensions in {:.2} s", - normalized.len(), - duration.as_secs_f32() - ); - assert!( !normalized.is_empty(), "embedding should have at least one dimension" diff --git a/llama-cpp-bindings-tests/tests/embedding_row_width.rs b/llama-cpp-bindings-tests/tests/embedding_row_width.rs index af80617a2..49d3ccb78 100644 --- a/llama-cpp-bindings-tests/tests/embedding_row_width.rs +++ b/llama-cpp-bindings-tests/tests/embedding_row_width.rs @@ -3,7 +3,7 @@ use llama_cpp_bindings::context::params::LlamaPoolingType; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; -use crate::context_after_decoding_text::context_after_decoding_text; +use llama_cpp_bindings_tests::context_after_decoding_text::context_after_decoding_text; const LFM2_COLBERT_OUTPUT_PROJECTION_WIDTH: usize = 128; const QWEN3_RERANKER_CLASSIFIER_LABELS: usize = 2; @@ -18,7 +18,7 @@ const QWEN3_RERANKER_CLASSIFIER_LABELS: usize = 2; embeddings = true, )] fn token_embeddings_are_as_wide_as_the_output_projection(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = context_after_decoding_text(fixture, LlamaPoolingType::None)?; + let context = context_after_decoding_text(fixture, LlamaPoolingType::None); assert_eq!( context.embeddings_ith(0)?.len(), @@ -40,7 +40,7 @@ fn token_embeddings_are_as_wide_as_the_output_projection(fixture: &LlamaFixture< fn pooled_sequence_embeddings_are_as_wide_as_the_output_projection( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean)?; + let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean); assert_eq!( context.embeddings_seq_ith(0)?.len(), @@ -62,7 +62,7 @@ fn pooled_sequence_embeddings_are_as_wide_as_the_output_projection( fn rank_pooled_sequence_embeddings_hold_one_score_per_classifier_label( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let context = context_after_decoding_text(fixture, LlamaPoolingType::Rank)?; + let context = context_after_decoding_text(fixture, LlamaPoolingType::Rank); assert_eq!( context.embeddings_seq_ith(0)?.len(), diff --git a/llama-cpp-bindings-tests/tests/multimodal_vision_inference.rs b/llama-cpp-bindings-tests/tests/image_question_answering.rs similarity index 91% rename from llama-cpp-bindings-tests/tests/multimodal_vision_inference.rs rename to llama-cpp-bindings-tests/tests/image_question_answering.rs index 2f90bf58c..a8d53deb2 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_vision_inference.rs +++ b/llama-cpp-bindings-tests/tests/image_question_answering.rs @@ -26,7 +26,9 @@ use llama_cpp_test_harness::llama_test; n_ubatch = 512, mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), )] -fn multimodal_vision_inference_produces_output(fixture: &LlamaFixture<'_>) -> Result<()> { +fn a_question_about_an_image_is_answered_with_its_image_tokens_counted_as_input( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let model = fixture.model; let mtmd_ctx = fixture .mtmd_context @@ -71,14 +73,6 @@ fn multimodal_vision_inference_produces_output(fixture: &LlamaFixture<'_>) -> Re let expected = ChunkTokenBreakdown::from_chunks(&chunks); - eprintln!( - "tokenized into {} chunks, text {} image {} audio {}", - chunks.len(), - expected.text, - expected.image, - expected.audio - ); - assert!( expected.image > 0, "vision input must produce at least one image chunk" @@ -100,8 +94,6 @@ fn multimodal_vision_inference_produces_output(fixture: &LlamaFixture<'_>) -> Re ) .with_context(|| "failed to evaluate chunks")?; - eprintln!("evaluated chunks, n_past = {n_past}"); - { let usage = classifier.usage(); assert_eq!(usage.prompt_tokens, expected.text); @@ -121,8 +113,6 @@ fn multimodal_vision_inference_produces_output(fixture: &LlamaFixture<'_>) -> Re } .run(); - eprintln!("generated text: {}", totals.generated.text()); - assert!( !totals.generated.text().is_empty(), "model should generate at least one token from image input" diff --git a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs index 9feaaf5b1..15f310ea6 100644 --- a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs +++ b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs @@ -1,5 +1,3 @@ -use std::io::Write; - use anyhow::Context as _; use anyhow::Result; use llama_cpp_bindings::SampledToken; @@ -190,8 +188,6 @@ fn json_schema_constrains_output(fixture: &LlamaFixture<'_>) -> Result<()> { let output_string = model.token_to_piece(&token, &mut decoder, true, None)?; generated.push_str(&output_string); - print!("{output_string}"); - std::io::stdout().flush()?; batch.clear(); batch.add(&token, n_cur, &[0], true)?; @@ -199,8 +195,6 @@ fn json_schema_constrains_output(fixture: &LlamaFixture<'_>) -> Result<()> { ctx.decode(&batch)?; } - println!(); - let parsed = serde_json::Deserializer::from_str(&generated) .into_iter::() .next() diff --git a/llama-cpp-bindings-tests/tests/logits_availability.rs b/llama-cpp-bindings-tests/tests/logits_availability.rs index b2e399c48..be9c40fcc 100644 --- a/llama-cpp-bindings-tests/tests/logits_availability.rs +++ b/llama-cpp-bindings-tests/tests/logits_availability.rs @@ -8,7 +8,7 @@ use llama_cpp_bindings::token::data_array::LlamaTokenDataArray; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; -use crate::context_after_decoding_text::context_after_decoding_text; +use llama_cpp_bindings_tests::context_after_decoding_text::context_after_decoding_text; #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), @@ -19,8 +19,8 @@ use crate::context_after_decoding_text::context_after_decoding_text; n_ubatch = 512, embeddings = true, )] -fn a_pooled_embeddings_decode_produces_no_logits(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean)?; +fn a_pooled_embeddings_decode_produces_no_logits(fixture: &LlamaFixture<'_>) { + let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean); assert_eq!( context.get_logits_ith(-1), @@ -28,8 +28,6 @@ fn a_pooled_embeddings_decode_produces_no_logits(fixture: &LlamaFixture<'_>) -> pooling_type: LlamaPoolingType::Mean, }) ); - - Ok(()) } #[llama_test( @@ -44,7 +42,7 @@ fn a_pooled_embeddings_decode_produces_no_logits(fixture: &LlamaFixture<'_>) -> fn sampling_after_a_pooled_embeddings_decode_is_refused_instead_of_aborting( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean)?; + let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean); let mut sampler = LlamaSampler::greedy()?; assert_eq!( @@ -70,15 +68,13 @@ fn sampling_after_a_pooled_embeddings_decode_is_refused_instead_of_aborting( )] fn a_model_whose_embeddings_graph_has_no_logits_head_produces_no_logits( fixture: &LlamaFixture<'_>, -) -> Result<()> { - let context = context_after_decoding_text(fixture, LlamaPoolingType::None)?; +) { + let context = context_after_decoding_text(fixture, LlamaPoolingType::None); assert_eq!( context.get_logits_ith(-1), Err(LogitsError::GraphProducesNoLogits) ); - - Ok(()) } #[llama_test( @@ -90,10 +86,8 @@ fn a_model_whose_embeddings_graph_has_no_logits_head_produces_no_logits( n_ubatch = 512, embeddings = true, )] -fn every_logits_accessor_refuses_a_pooled_embeddings_decode( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean)?; +fn every_logits_accessor_refuses_a_pooled_embeddings_decode(fixture: &LlamaFixture<'_>) { + let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean); assert_eq!( [ @@ -112,8 +106,6 @@ fn every_logits_accessor_refuses_a_pooled_embeddings_decode( }) }) ); - - Ok(()) } #[llama_test( @@ -128,7 +120,7 @@ fn every_logits_accessor_refuses_a_pooled_embeddings_decode( fn classified_sampling_after_a_pooled_embeddings_decode_is_refused( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean)?; + let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean); let mut classifier = SampledTokenClassifier::undeterminable(fixture.model); assert_eq!( diff --git a/llama-cpp-bindings-tests/tests/main.rs b/llama-cpp-bindings-tests/tests/main.rs index f69e45a48..e2a63740d 100644 --- a/llama-cpp-bindings-tests/tests/main.rs +++ b/llama-cpp-bindings-tests/tests/main.rs @@ -8,7 +8,6 @@ mod chat_session_llama_cpp_failures; mod chat_template_lookup; mod chat_template_tool_call_support; mod completion_usage_accounting; -mod context_after_decoding_text; mod context_creation_limits; mod context_dimensions; mod context_synchronization; @@ -28,6 +27,7 @@ mod end_of_generation_token_detection; mod generation_end_classification; mod grammar_sampler; mod hidden_state_extraction; +mod image_question_answering; mod json_schema_constrained_generation; mod kv_cache_memory_availability; mod kv_cache_occupancy; @@ -59,7 +59,6 @@ mod multimodal_projector_loading; mod multimodal_prompt_parts_tokenization; mod multimodal_reasoning_classification; mod multimodal_usage_accounting; -mod multimodal_vision_inference; mod parameter_fitting; mod performance_timings; mod prompt_token_accounting; @@ -76,7 +75,6 @@ mod threadpool_detachment; mod token_attributes; mod token_candidates; mod tokenization; -mod tokenize_synthetic; mod tool_call_classification; mod tool_call_grammar; mod tool_call_parsing; diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs index a0ab644a1..71d7f51b1 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs @@ -26,7 +26,7 @@ use llama_cpp_fault_injection::ArmedFault; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; -use crate::tokenize_synthetic::tokenize_synthetic; +use llama_cpp_bindings_tests::tokenize_prompt_with_synthetic_image::tokenize_prompt_with_synthetic_image; fn eval_synthetic_bitmap(fixture: &LlamaFixture<'_>, width: u32, height: u32) -> Result<()> { let mtmd_ctx = fixture @@ -150,7 +150,7 @@ fn eval_chunks_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_> let mtmd_ctx = fixture .mtmd_context .expect("mmproj_file declared in attribute"); - let chunks = tokenize_synthetic(fixture, "Describe: <__media__>")?; + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Describe: <__media__>"); let mut llama_ctx = LlamaContext::from_model( fixture.model, fixture.backend, @@ -230,7 +230,7 @@ fn eval_single_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_> (*fixture.context_params).into_llama_context_params(), )?; let fitted_chunks = FittedMtmdInputChunks::fit( - tokenize_synthetic(fixture, "Describe: <__media__>")?, + tokenize_prompt_with_synthetic_image(fixture, "Describe: <__media__>"), mtmd_ctx, llama_ctx.n_ubatch(), )?; @@ -263,7 +263,7 @@ fn eval_single_rejects_a_batch_larger_than_the_context_batch_before_evaluating( .expect("mmproj_file declared in attribute"); let mut llama_ctx = fixture.build_context()?; let fitted_chunks = FittedMtmdInputChunks::fit( - tokenize_synthetic(fixture, "Describe: <__media__>")?, + tokenize_prompt_with_synthetic_image(fixture, "Describe: <__media__>"), mtmd_ctx, llama_ctx.n_ubatch(), )?; @@ -300,7 +300,7 @@ fn eval_chunks_rejects_a_sequence_the_context_does_not_have_instead_of_aborting( .mtmd_context .expect("mmproj_file declared in attribute"); let mut llama_ctx = fixture.build_context()?; - let chunks = tokenize_synthetic(fixture, "Describe: <__media__>")?; + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Describe: <__media__>"); let n_seq_max = llama_ctx.n_seq_max(); let missing_sequence = i32::try_from(n_seq_max)?; let n_batch = i32::try_from(llama_ctx.n_batch())?; @@ -348,7 +348,7 @@ fn eval_single_rejects_a_micro_batch_smaller_than_the_fitted_one_before_evaluati )?; let fitted_micro_batch_tokens = llama_ctx.n_ubatch(); let fitted_chunks = FittedMtmdInputChunks::fit( - tokenize_synthetic(fixture, "Describe: <__media__>")?, + tokenize_prompt_with_synthetic_image(fixture, "Describe: <__media__>"), mtmd_ctx, fitted_micro_batch_tokens, )?; @@ -397,7 +397,7 @@ fn classifier_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_>) let mtmd_ctx = fixture .mtmd_context .expect("mmproj_file declared in attribute"); - let chunks = tokenize_synthetic(fixture, "Describe: <__media__>")?; + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Describe: <__media__>"); let mut llama_ctx = LlamaContext::from_model( fixture.model, fixture.backend, diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs index e5ef6abdf..6423a522f 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs @@ -11,7 +11,7 @@ use llama_cpp_fault_injection::injected_fault_message; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; -use crate::tokenize_synthetic::tokenize_synthetic; +use llama_cpp_bindings_tests::tokenize_prompt_with_synthetic_image::tokenize_prompt_with_synthetic_image; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), @@ -32,7 +32,7 @@ use crate::tokenize_synthetic::tokenize_synthetic; mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), )] fn text_chunk_has_text_type(fixture: &LlamaFixture<'_>) -> Result<()> { - let chunks = tokenize_synthetic(fixture, "Hello world <__media__>")?; + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Hello world <__media__>"); let first_chunk = chunks.get(0).context("missing first chunk")?; assert_eq!(first_chunk.chunk_type()?, MtmdInputChunkType::Text); Ok(()) @@ -57,7 +57,7 @@ fn text_chunk_has_text_type(fixture: &LlamaFixture<'_>) -> Result<()> { mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), )] fn text_chunk_returns_text_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { - let chunks = tokenize_synthetic(fixture, "Hello world <__media__>")?; + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Hello world <__media__>"); let first_chunk = chunks.get(0).context("missing first chunk")?; let tokens = first_chunk .text_tokens_of(first_chunk.chunk_type()?) @@ -87,7 +87,7 @@ fn text_chunk_returns_text_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), )] fn chunk_n_tokens_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { - let chunks = tokenize_synthetic(fixture, "Hello world <__media__>")?; + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Hello world <__media__>"); let first_chunk = chunks.get(0).context("missing first chunk")?; assert!(first_chunk.n_tokens() > 0); Ok(()) @@ -112,7 +112,7 @@ fn chunk_n_tokens_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), )] fn chunk_n_positions_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { - let chunks = tokenize_synthetic(fixture, "Hello world <__media__>")?; + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Hello world <__media__>"); let first_chunk = chunks.get(0).context("missing first chunk")?; assert!(first_chunk.n_positions() > 0); Ok(()) @@ -137,7 +137,7 @@ fn chunk_n_positions_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), )] fn text_chunk_id_returns_none(fixture: &LlamaFixture<'_>) -> Result<()> { - let chunks = tokenize_synthetic(fixture, "Hello <__media__>")?; + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Hello <__media__>"); let first_chunk = chunks.get(0).context("missing first chunk")?; assert_eq!(first_chunk.chunk_type()?, MtmdInputChunkType::Text); assert!(first_chunk.id().is_none()); @@ -163,7 +163,7 @@ fn text_chunk_id_returns_none(fixture: &LlamaFixture<'_>) -> Result<()> { mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), )] fn image_chunk_returns_none_for_text_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { - let chunks = tokenize_synthetic(fixture, "Hello <__media__>")?; + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Hello <__media__>"); for chunk_index in 0..chunks.len() { let chunk = chunks .get(chunk_index) @@ -196,7 +196,7 @@ fn image_chunk_returns_none_for_text_tokens(fixture: &LlamaFixture<'_>) -> Resul mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), )] fn image_chunk_id_returns_some(fixture: &LlamaFixture<'_>) -> Result<()> { - let chunks = tokenize_synthetic(fixture, "Hello <__media__>")?; + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Hello <__media__>"); for chunk_index in 0..chunks.len() { let chunk = chunks .get(chunk_index) diff --git a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs index bf1118a13..e370b61c1 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs @@ -66,7 +66,9 @@ fn load_fixture_bitmap(mtmd_ctx: &MtmdContext, file_name: &str) -> Result) -> Result<()> { +fn an_image_and_audio_prompt_is_answered_with_both_media_counted_as_input( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let model = fixture.model; let mtmd_ctx = fixture .mtmd_context diff --git a/llama-cpp-bindings-tests/tests/reranking.rs b/llama-cpp-bindings-tests/tests/reranking.rs index 75253c997..0eea7acbb 100644 --- a/llama-cpp-bindings-tests/tests/reranking.rs +++ b/llama-cpp-bindings-tests/tests/reranking.rs @@ -1,12 +1,9 @@ -use std::time::Duration; - use anyhow::Context; use anyhow::Result; use anyhow::bail; use llama_cpp_bindings::SampledTokenClassifier; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; -use llama_cpp_bindings::ggml_time_us; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::AddBos; use llama_cpp_bindings::model::ParseSpecialTokens; @@ -75,7 +72,6 @@ fn reranking_produces_scores(fixture: &LlamaFixture<'_>) -> Result<()> { let mut classifier = SampledTokenClassifier::undeterminable(model); let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); - let t_main_start = ggml_time_us(); for (sequence_index, tokens) in tokens_lines_list.iter().enumerate() { classifier.feed_prompt_sequence_to_batch( @@ -108,16 +104,6 @@ fn reranking_produces_scores(fixture: &LlamaFixture<'_>) -> Result<()> { embeddings.push(normalize(raw_embedding)); } - let t_main_end = ggml_time_us(); - let duration = Duration::from_micros(u64::try_from(t_main_end - t_main_start)?); - - let tokens_per_second = f64::from(u32::try_from(total_tokens)?) / duration.as_secs_f64(); - - eprintln!( - "created embeddings for {total_tokens} tokens in {:.2} s, speed {tokens_per_second:.2} t/s", - duration.as_secs_f64(), - ); - assert_eq!( embeddings.len(), document_count, @@ -132,7 +118,6 @@ fn reranking_produces_scores(fixture: &LlamaFixture<'_>) -> Result<()> { } let similarity = cosine_similarity(&embeddings[0], &embeddings[1]); - eprintln!("cosine similarity between document embeddings: {similarity:.4}"); assert!( similarity.is_finite(), From 18035f551282558d221e33f0853c42f5696913f1 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sat, 31 Oct 2026 04:41:39 +0100 Subject: [PATCH 21/82] Hide superseded llama.cpp APIs from the bindings, drop the perf and row-0 logits APIs, and fix IMROPE, NUMA init, buffer override ownership and chunk evaluation exceptions --- .../fixtures/bindings/llama_types.h | 5 + .../src/binding_callbacks.rs | 35 +- .../src/bindings_generation.rs | 30 +- llama-cpp-bindings-sys/wrapper_context.cpp | 4 + llama-cpp-bindings-sys/wrapper_context.h | 2 + .../wrapper_fault_injection.h | 1 + llama-cpp-bindings-sys/wrapper_mtmd.cpp | 2 + .../src/penalized_nucleus_sampler.rs | 2 +- .../tests/completion_usage_accounting.rs | 3 +- .../tests/grammar_sampler.rs | 6 +- .../json_schema_constrained_generation.rs | 4 +- .../tests/llguidance_sampler.rs | 6 +- .../tests/logits_availability.rs | 6 +- .../tests/logits_retrieval.rs | 29 -- llama-cpp-bindings-tests/tests/main.rs | 1 - .../tests/model_architecture.rs | 10 +- .../tests/multimodal_chunk_evaluation.rs | 32 ++ .../multimodal_reasoning_classification.rs | 6 +- .../tests/performance_timings.rs | 26 -- .../tests/sampler_token_acceptance.rs | 8 +- .../tests/token_candidates.rs | 56 ---- .../tests/tool_call_grammar.rs | 2 +- .../tests/unconstrained_token_sampling.rs | 9 +- llama-cpp-bindings/src/context.rs | 49 +-- llama-cpp-bindings/src/lib.rs | 1 - llama-cpp-bindings/src/llama_backend.rs | 17 +- llama-cpp-bindings/src/llama_token_attrs.rs | 4 +- llama-cpp-bindings/src/model.rs | 8 +- llama-cpp-bindings/src/model/params.rs | 26 +- llama-cpp-bindings/src/model/rope_type.rs | 10 + llama-cpp-bindings/src/mtmd.rs | 1 + llama-cpp-bindings/src/mtmd/eval_chunk.rs | 310 ++++++++++++++++++ .../src/mtmd/fitted_mtmd_input_chunk.rs | 300 +---------------- .../src/mtmd/mtmd_input_chunks.rs | 69 ++-- llama-cpp-bindings/src/sampling.rs | 24 +- llama-cpp-bindings/src/timing.rs | 210 ------------ .../tests/sampler_chain_without_backend.rs | 2 +- 37 files changed, 542 insertions(+), 774 deletions(-) delete mode 100644 llama-cpp-bindings-tests/tests/performance_timings.rs create mode 100644 llama-cpp-bindings/src/mtmd/eval_chunk.rs delete mode 100644 llama-cpp-bindings/src/timing.rs diff --git a/llama-cpp-bindings-build/fixtures/bindings/llama_types.h b/llama-cpp-bindings-build/fixtures/bindings/llama_types.h index 2b2cd503c..9f275522b 100644 --- a/llama-cpp-bindings-build/fixtures/bindings/llama_types.h +++ b/llama-cpp-bindings-build/fixtures/bindings/llama_types.h @@ -13,3 +13,8 @@ struct llama_token_data { struct llama_sampler_chain_params { int no_perf; }; + +enum ggml_prec { + GGML_PREC_UNDEFINED = 0, + GGML_PREC_DEFAULT = 0, +}; diff --git a/llama-cpp-bindings-build/src/binding_callbacks.rs b/llama-cpp-bindings-build/src/binding_callbacks.rs index 38b967646..91ed27767 100644 --- a/llama-cpp-bindings-build/src/binding_callbacks.rs +++ b/llama-cpp-bindings-build/src/binding_callbacks.rs @@ -5,6 +5,8 @@ use std::rc::Rc; use bindgen::FieldVisibilityKind; use bindgen::callbacks::DeriveInfo; +use bindgen::callbacks::EnumVariantCustomBehavior; +use bindgen::callbacks::EnumVariantValue; use bindgen::callbacks::FieldInfo; use bindgen::callbacks::ParseCallbacks; @@ -14,6 +16,8 @@ use crate::privatized_field::PrivatizedField; const COMPARED_STRUCTS: &[&str] = &["llama_logit_bias", "llama_token_data"]; +const SUPERSEDED_ENUM_VARIANTS: &[&str] = &["GGML_PREC_DEFAULT"]; + fn derives_for(type_name: &str) -> Vec { if COMPARED_STRUCTS.contains(&type_name) { vec!["PartialEq".to_owned()] @@ -22,8 +26,15 @@ fn derives_for(type_name: &str) -> Vec { } } -/// Observes bindgen while it parses: keeps the [`PrivatizedField::ALL`] fields private, derives -/// `PartialEq` for the structs the bindings compare, and records what the bindings depend on. +fn variant_behavior_for(variant_name: &str) -> Option { + SUPERSEDED_ENUM_VARIANTS + .contains(&variant_name) + .then_some(EnumVariantCustomBehavior::Hide) +} + +/// Observes bindgen while it parses: keeps the [`PrivatizedField::ALL`] fields private, hides +/// enum variants llama.cpp superseded, derives `PartialEq` for the structs the bindings compare, +/// and records what the bindings depend on. #[derive(Clone, Debug)] pub struct BindingCallbacks { privatized_field_hits: Rc<[Cell]>, @@ -90,6 +101,15 @@ impl ParseCallbacks for BindingCallbacks { derives_for(info.name) } + fn enum_variant_behavior( + &self, + _enum_name: Option<&str>, + original_variant_name: &str, + _variant_value: EnumVariantValue, + ) -> Option { + variant_behavior_for(original_variant_name) + } + fn field_visibility(&self, info: FieldInfo<'_>) -> Option { PrivatizedField::ALL .iter() @@ -107,7 +127,18 @@ impl ParseCallbacks for BindingCallbacks { #[cfg(test)] mod tests { + use bindgen::callbacks::EnumVariantCustomBehavior; + use super::derives_for; + use super::variant_behavior_for; + + #[test] + fn only_superseded_enum_variants_are_hidden() { + assert_eq!( + ["GGML_PREC_DEFAULT", "GGML_PREC_UNDEFINED"].map(variant_behavior_for), + [Some(EnumVariantCustomBehavior::Hide), None] + ); + } #[test] fn only_the_structs_the_bindings_compare_derive_partial_eq() { diff --git a/llama-cpp-bindings-build/src/bindings_generation.rs b/llama-cpp-bindings-build/src/bindings_generation.rs index 93003ed48..6706ca65c 100644 --- a/llama-cpp-bindings-build/src/bindings_generation.rs +++ b/llama-cpp-bindings-build/src/bindings_generation.rs @@ -63,6 +63,26 @@ const DEPRECATED_FUNCTIONS: &[&str] = &[ "mtmd_image_tokens_get_ny", ]; +const SUPERSEDED_ITEMS: &[&str] = &[ + "LLAMA_TOKEN_TYPE_.*", + "ggml_backend_offload_op", + "ggml_backend_supports_buft", + "ggml_backend_supports_op", + "llama_batch", + "llama_batch_free", + "llama_batch_get_one", + "llama_batch_init", + "llama_decode", + "llama_encode", + "llama_get_embeddings", + "llama_get_logits", + "llama_perf_.*", + "llama_token_type", + "mtmd_image_tokens_get_id", + "mtmd_image_tokens_get_n_pos", + "mtmd_image_tokens_get_n_tokens", +]; + fn configured_builder() -> Builder { let builder = Builder::default() .generate_comments(false) @@ -86,11 +106,13 @@ fn configured_builder() -> Builder { .blocklist_type("_IO_.*") .prepend_enum_name(false); - DEPRECATED_FUNCTIONS + let builder = DEPRECATED_FUNCTIONS + .iter() + .fold(builder, Builder::blocklist_function); + + SUPERSEDED_ITEMS .iter() - .fold(builder, |builder, function| { - builder.blocklist_function(function) - }) + .fold(builder, Builder::blocklist_item) } fn fault_injection_args(environment: &BuildEnvironment) -> Vec { diff --git a/llama-cpp-bindings-sys/wrapper_context.cpp b/llama-cpp-bindings-sys/wrapper_context.cpp index 865deb5f1..a07d50df4 100644 --- a/llama-cpp-bindings-sys/wrapper_context.cpp +++ b/llama-cpp-bindings-sys/wrapper_context.cpp @@ -84,6 +84,10 @@ extern "C" auto llama_rs_model_causal_attn(const struct llama_model * model) -> return model->hparams.causal_attn; } +extern "C" auto llama_rs_context_produced_logits(struct llama_context * ctx) -> bool { + return llama_get_logits(ctx) != nullptr; +} + extern "C" auto llama_rs_get_logits_ith(struct llama_context * ctx, int32_t token_index) -> float * { if (llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_GET_LOGITS_ITH_RETURNS_NULL)) { return nullptr; diff --git a/llama-cpp-bindings-sys/wrapper_context.h b/llama-cpp-bindings-sys/wrapper_context.h index 7a9b9030e..90db33852 100644 --- a/llama-cpp-bindings-sys/wrapper_context.h +++ b/llama-cpp-bindings-sys/wrapper_context.h @@ -40,6 +40,8 @@ llama_rs_context_nextn_hidden_state_probe_status llama_rs_context_graph_produces bool llama_rs_model_causal_attn(const struct llama_model * model); +bool llama_rs_context_produced_logits(struct llama_context * ctx); + float * llama_rs_get_logits_ith(struct llama_context * ctx, int32_t token_index); int32_t llama_rs_set_adapters_lora( diff --git a/llama-cpp-bindings-sys/wrapper_fault_injection.h b/llama-cpp-bindings-sys/wrapper_fault_injection.h index 4fd53f904..3adc2009a 100644 --- a/llama-cpp-bindings-sys/wrapper_fault_injection.h +++ b/llama-cpp-bindings-sys/wrapper_fault_injection.h @@ -19,6 +19,7 @@ typedef enum llama_rs_fault_point { LLAMA_RS_FAULT_POINT_CHAT_SESSION_PROMPT_PART_TEXT_NOT_UTF8, LLAMA_RS_FAULT_POINT_GET_LOGITS_ITH_RETURNS_NULL, LLAMA_RS_FAULT_POINT_LLGUIDANCE_FACTORY_FAILS, + LLAMA_RS_FAULT_POINT_MTMD_EVAL_CHUNK_SINGLE_THROWS, LLAMA_RS_FAULT_POINT_MTMD_GET_MEMORY_USAGE_THROWS, LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNK_TYPE_UNKNOWN, LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNKS_GET_RETURNS_NULL, diff --git a/llama-cpp-bindings-sys/wrapper_mtmd.cpp b/llama-cpp-bindings-sys/wrapper_mtmd.cpp index 33c51a302..79637a3a2 100644 --- a/llama-cpp-bindings-sys/wrapper_mtmd.cpp +++ b/llama-cpp-bindings-sys/wrapper_mtmd.cpp @@ -295,6 +295,8 @@ extern "C" auto llama_rs_mtmd_eval_chunk_single( } try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_MTMD_EVAL_CHUNK_SINGLE_THROWS); + int32_t const result = mtmd_helper_eval_chunk_single( ctx, lctx, chunk, n_past, seq_id, n_batch, logits_last, out_new_n_past); if (result != 0) { diff --git a/llama-cpp-bindings-tests/src/penalized_nucleus_sampler.rs b/llama-cpp-bindings-tests/src/penalized_nucleus_sampler.rs index d0581c09b..4bd75ec9e 100644 --- a/llama-cpp-bindings-tests/src/penalized_nucleus_sampler.rs +++ b/llama-cpp-bindings-tests/src/penalized_nucleus_sampler.rs @@ -11,7 +11,7 @@ const SAMPLING_SEED: u32 = 0x00C0_FFEE; /// Panics when llama.cpp cannot build one of the samplers. #[must_use] pub fn penalized_nucleus_sampler(model: &LlamaModel) -> LlamaSampler { - LlamaSampler::chain_simple([ + LlamaSampler::chain([ LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0) .expect("the penalties sampler must build"), LlamaSampler::top_k(40).expect("the top-k sampler must build"), diff --git a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs index f644ccf34..9b21257d3 100644 --- a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs +++ b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs @@ -83,8 +83,7 @@ fn raw_prompt_completion_usage_matches_the_classified_tokens( assert_eq!(promoted, prompt_token_count); assert_eq!(classifier.usage().prompt_tokens, prompt_token_count); - let mut sampler = - LlamaSampler::chain_simple([LlamaSampler::dist(1234)?, LlamaSampler::greedy()?])?; + let mut sampler = LlamaSampler::chain([LlamaSampler::dist(1234)?, LlamaSampler::greedy()?])?; let initial_position = batch.n_tokens(); let outcome = ClassifySampleLoop { classifier: &mut classifier, diff --git a/llama-cpp-bindings-tests/tests/grammar_sampler.rs b/llama-cpp-bindings-tests/tests/grammar_sampler.rs index 13abdb38a..c4e42f566 100644 --- a/llama-cpp-bindings-tests/tests/grammar_sampler.rs +++ b/llama-cpp-bindings-tests/tests/grammar_sampler.rs @@ -68,7 +68,7 @@ fn grammar_sampler_constrains_output_to_yes_or_no(fixture: &LlamaFixture<'_>) -> context.decode(&batch)?; - let mut sampler = LlamaSampler::chain_simple([ + let mut sampler = LlamaSampler::chain([ LlamaSampler::grammar(model, r"root ::= [Yy] [Ee] [Ss] | [Nn] [Oo]", "root")?, LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?, @@ -167,7 +167,7 @@ fn sample_with_grammar_produces_constrained_output_in_loop( context.decode(&batch)?; classifier.commit_prompt_tokens(); - let mut sampler = LlamaSampler::chain_simple([ + let mut sampler = LlamaSampler::chain([ LlamaSampler::grammar(model, r#"root ::= "yes" | "no""#, "root")?, LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?, @@ -253,7 +253,7 @@ fn qwen35_grammar_with_a_token_reference_constrains_the_first_token( batch.add_sequence(&prompt_tokens, 0, false)?; context.decode(&batch)?; - let mut sampler = LlamaSampler::chain_simple([ + let mut sampler = LlamaSampler::chain([ LlamaSampler::grammar(model, r#"root ::= "x""#, "root")?, LlamaSampler::greedy()?, ])?; diff --git a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs index 15f310ea6..5cb243d59 100644 --- a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs +++ b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs @@ -64,7 +64,7 @@ fn json_schema_grammar_sampler_constrains_output_to_json(fixture: &LlamaFixture< r#"{"type": "object", "properties": {"answer": {"type": "string"}}, "required": ["answer"]}"#, )?; - let mut sampler = LlamaSampler::chain_simple([ + let mut sampler = LlamaSampler::chain([ LlamaSampler::grammar(model, &grammar_str, "root")?, LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?, @@ -173,7 +173,7 @@ fn json_schema_constrains_output(fixture: &LlamaFixture<'_>) -> Result<()> { }"#; let llg_sampler = LlamaSampler::llguidance(model, "json", schema)?; - let mut sampler = LlamaSampler::chain_simple([llg_sampler, LlamaSampler::greedy()?])?; + let mut sampler = LlamaSampler::chain([llg_sampler, LlamaSampler::greedy()?])?; let mut n_cur = batch.n_tokens(); let mut decoder = encoding_rs::UTF_8.new_decoder(); diff --git a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs index f0af41d09..76609d201 100644 --- a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs +++ b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs @@ -417,7 +417,7 @@ fn samples_token_constrained_by_grammar(fixture: &LlamaFixture<'_>) -> Result<() context.decode(&batch)?; let llg_sampler = create_llg_sampler(model, "regex", REGEX_GRAMMAR)?; - let mut chain = LlamaSampler::chain_simple([llg_sampler, LlamaSampler::greedy()?])?; + let mut chain = LlamaSampler::chain([llg_sampler, LlamaSampler::greedy()?])?; let token = chain.sample(&context, batch.n_tokens() - 1)?; assert!( @@ -602,7 +602,7 @@ fn llguidance_chain_samples_a_valid_token(fixture: &LlamaFixture<'_>) -> Result< context.decode(&batch)?; let llg_sampler = create_llg_sampler(model, "regex", REGEX_GRAMMAR)?; - let mut chain = LlamaSampler::chain_simple([llg_sampler, LlamaSampler::greedy()?])?; + let mut chain = LlamaSampler::chain([llg_sampler, LlamaSampler::greedy()?])?; let token = chain.sample(&context, batch.n_tokens() - 1)?; assert!(token.0 >= 0); @@ -632,7 +632,7 @@ fn sampling_through_a_grammar_in_an_error_state_reports_the_grammar_failure( batch.add_sequence(&tokens, 0, false)?; context.decode(&batch)?; - let mut chain = LlamaSampler::chain_simple([ + let mut chain = LlamaSampler::chain([ create_llg_sampler(model, "regex", REGEX_GRAMMAR)?, LlamaSampler::greedy()?, ])?; diff --git a/llama-cpp-bindings-tests/tests/logits_availability.rs b/llama-cpp-bindings-tests/tests/logits_availability.rs index be9c40fcc..c267ed02b 100644 --- a/llama-cpp-bindings-tests/tests/logits_availability.rs +++ b/llama-cpp-bindings-tests/tests/logits_availability.rs @@ -91,16 +91,14 @@ fn every_logits_accessor_refuses_a_pooled_embeddings_decode(fixture: &LlamaFixtu assert_eq!( [ - context.get_logits().err(), - context.candidates().err(), - context.token_data_array().err(), + context.get_logits_ith(-1).err(), context.candidates_ith(-1).err(), context.token_data_array_ith(-1).err(), context .fill_token_data_array_ith(-1, &mut LlamaTokenDataArray::new(Vec::new(), false)) .err(), ], - [(); 6].map(|()| { + [(); 4].map(|()| { Some(LogitsError::PooledEmbeddingsProduceNoLogits { pooling_type: LlamaPoolingType::Mean, }) diff --git a/llama-cpp-bindings-tests/tests/logits_retrieval.rs b/llama-cpp-bindings-tests/tests/logits_retrieval.rs index ba70f7bd7..a08ace11a 100644 --- a/llama-cpp-bindings-tests/tests/logits_retrieval.rs +++ b/llama-cpp-bindings-tests/tests/logits_retrieval.rs @@ -9,35 +9,6 @@ use llama_cpp_fault_injection::ArmedFault; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn get_logits_returns_values_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - - let decode_result = context.decode(&batch); - assert!(decode_result.is_ok()); - - let logits = context.get_logits()?; - assert_ne!(logits, Vec::::new()); - - Ok(()) -} - #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, diff --git a/llama-cpp-bindings-tests/tests/main.rs b/llama-cpp-bindings-tests/tests/main.rs index e2a63740d..47f0868e8 100644 --- a/llama-cpp-bindings-tests/tests/main.rs +++ b/llama-cpp-bindings-tests/tests/main.rs @@ -60,7 +60,6 @@ mod multimodal_prompt_parts_tokenization; mod multimodal_reasoning_classification; mod multimodal_usage_accounting; mod parameter_fitting; -mod performance_timings; mod prompt_token_accounting; mod reasoning_classification; mod reranking; diff --git a/llama-cpp-bindings-tests/tests/model_architecture.rs b/llama-cpp-bindings-tests/tests/model_architecture.rs index 9c614efee..190087660 100644 --- a/llama-cpp-bindings-tests/tests/model_architecture.rs +++ b/llama-cpp-bindings-tests/tests/model_architecture.rs @@ -127,10 +127,8 @@ fn rope_type_returns_a_known_variant_for_rope_carrying_default_models(fixture: & n_batch = 128, n_ubatch = 64, )] -fn rope_type_returns_none_for_qwen_default_models(fixture: &LlamaFixture<'_>) { - let rope = fixture.model.rope_type(); - assert!( - rope.is_none(), - "Qwen 3.5 and Qwen 3.6 default GGUFs do not expose a rope_type in their metadata; got {rope:?}" - ); +fn rope_type_reports_interleaved_mrope_for_qwen_default_models(fixture: &LlamaFixture<'_>) { + use llama_cpp_bindings::model::rope_type::RopeType; + + assert_eq!(fixture.model.rope_type(), Some(RopeType::IMRope)); } diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs index 71d7f51b1..c593dc08b 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs @@ -23,6 +23,7 @@ use llama_cpp_bindings_tests::media_question_chat_session::media_question_chat_s use llama_cpp_bindings_tests::prime_kv_cache_with::prime_kv_cache_with; use llama_cpp_bindings_tests::prompt_question::PROMPT_QUESTION; use llama_cpp_fault_injection::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -128,6 +129,37 @@ fn eval_chunks_returns_batch_size_exceeds_context_limit_for_huge_batch( Ok(()) } +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_chunks_reports_an_exception_llama_cpp_throws_while_evaluating_a_chunk( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Describe: <__media__>"); + let mut llama_ctx = fixture.build_context()?; + let n_batch = i32::try_from(llama_ctx.n_batch())?; + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_MTMD_EVAL_CHUNK_SINGLE_THROWS); + + assert_eq!( + chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, true), + Err(MtmdEvalError::Reported { + message: injected_fault_message() + }) + ); + + Ok(()) +} + #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, diff --git a/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs index 6189c76d2..3f8825cc5 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs @@ -78,7 +78,7 @@ fn gemma4_classifier_emits_reasoning_for_multimodal_thinking_prompt( }, )?; - let mut sampler = LlamaSampler::chain_simple([ + let mut sampler = LlamaSampler::chain([ LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, LlamaSampler::top_k(40)?, LlamaSampler::top_p(0.9, 1)?, @@ -289,7 +289,7 @@ fn qwen35_classifier_emits_reasoning_for_multimodal_thinking_prompt( }, )?; - let mut sampler = LlamaSampler::chain_simple([ + let mut sampler = LlamaSampler::chain([ LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, LlamaSampler::top_k(40)?, LlamaSampler::top_p(0.9, 1)?, @@ -381,7 +381,7 @@ fn qwen36_classifier_emits_reasoning_for_multimodal_thinking_prompt( }, )?; - let mut sampler = LlamaSampler::chain_simple([ + let mut sampler = LlamaSampler::chain([ LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, LlamaSampler::top_k(40)?, LlamaSampler::top_p(0.9, 1)?, diff --git a/llama-cpp-bindings-tests/tests/performance_timings.rs b/llama-cpp-bindings-tests/tests/performance_timings.rs deleted file mode 100644 index 48613ca1b..000000000 --- a/llama-cpp-bindings-tests/tests/performance_timings.rs +++ /dev/null @@ -1,26 +0,0 @@ -use anyhow::Result; -use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn timings_after_reset_report_a_non_negative_start_time(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - context.reset_timings(); - let timings = context.timings(); - assert!(timings.t_start_ms() >= 0.0); - - Ok(()) -} diff --git a/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs b/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs index e18f86923..2300675a1 100644 --- a/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs +++ b/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs @@ -14,7 +14,7 @@ use llama_cpp_test_harness::llama_test; n_ubatch = 512, )] fn accept_many_iterates_over_borrowed_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut sampler = LlamaSampler::chain_simple([LlamaSampler::greedy()?])?; + let mut sampler = LlamaSampler::chain([LlamaSampler::greedy()?])?; let tokens = vec![ fixture .model @@ -40,7 +40,7 @@ fn accept_many_iterates_over_borrowed_tokens(fixture: &LlamaFixture<'_>) -> Resu n_ubatch = 512, )] fn with_tokens_returns_self_after_accepting_each_token(fixture: &LlamaFixture<'_>) -> Result<()> { - let sampler = LlamaSampler::chain_simple([LlamaSampler::greedy()?])?; + let sampler = LlamaSampler::chain([LlamaSampler::greedy()?])?; let tokens = [ fixture .model @@ -66,7 +66,7 @@ fn with_tokens_returns_self_after_accepting_each_token(fixture: &LlamaFixture<'_ n_ubatch = 512, )] fn accept_consumes_a_single_token(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut sampler = LlamaSampler::chain_simple([LlamaSampler::greedy()?])?; + let mut sampler = LlamaSampler::chain([LlamaSampler::greedy()?])?; sampler.accept( fixture @@ -87,7 +87,7 @@ fn accept_consumes_a_single_token(fixture: &LlamaFixture<'_>) -> Result<()> { n_ubatch = 512, )] fn accept_succeeds_for_a_valid_token(_fixture: &LlamaFixture<'_>) -> Result<()> { - let mut sampler = LlamaSampler::chain_simple([LlamaSampler::greedy()?])?; + let mut sampler = LlamaSampler::chain([LlamaSampler::greedy()?])?; sampler.accept(LlamaToken::new(0))?; diff --git a/llama-cpp-bindings-tests/tests/token_candidates.rs b/llama-cpp-bindings-tests/tests/token_candidates.rs index 2b0645b72..0d4f2f467 100644 --- a/llama-cpp-bindings-tests/tests/token_candidates.rs +++ b/llama-cpp-bindings-tests/tests/token_candidates.rs @@ -10,34 +10,6 @@ use llama_cpp_bindings::token::data_array::LlamaTokenDataArray; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn token_data_array_has_entries_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let token_data_array = context.token_data_array()?; - - assert_ne!(token_data_array.data, Vec::::new()); - - Ok(()) -} - #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -70,34 +42,6 @@ fn token_data_array_ith_returns_valid_data(fixture: &LlamaFixture<'_>) -> Result Ok(()) } -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn candidates_returns_n_vocab_entries(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let count = context.candidates()?.count(); - - assert_eq!(count, usize::try_from(fixture.model.n_vocab())?); - - Ok(()) -} - #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, diff --git a/llama-cpp-bindings-tests/tests/tool_call_grammar.rs b/llama-cpp-bindings-tests/tests/tool_call_grammar.rs index 646762d61..3d825c874 100644 --- a/llama-cpp-bindings-tests/tests/tool_call_grammar.rs +++ b/llama-cpp-bindings-tests/tests/tool_call_grammar.rs @@ -21,7 +21,7 @@ fn generated_tool_call_answer( let grammar_sampler = session .grammar_sampler()? .context("a tool request must constrain generation with a grammar")?; - let mut sampler = LlamaSampler::chain_simple([grammar_sampler, LlamaSampler::greedy()?])?; + let mut sampler = LlamaSampler::chain([grammar_sampler, LlamaSampler::greedy()?])?; let answer = ChatAnswerGeneration { fixture, session, diff --git a/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs b/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs index 6afe72098..60b60b702 100644 --- a/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs +++ b/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs @@ -58,8 +58,7 @@ fn sample_returns_result_and_succeeds_with_valid_index(fixture: &LlamaFixture<'_ context.decode(&batch)?; - let mut sampler = - LlamaSampler::chain_simple([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; + let mut sampler = LlamaSampler::chain([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; let result = sampler.sample(&context, batch.n_tokens() - 1); @@ -116,8 +115,7 @@ fn sample_without_grammar_produces_multiple_tokens(fixture: &LlamaFixture<'_>) - context.decode(&batch)?; - let mut sampler = - LlamaSampler::chain_simple([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; + let mut sampler = LlamaSampler::chain([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; let mut classifier = SampledTokenClassifier::undeterminable(model); let mut outcomes = Vec::new(); @@ -204,8 +202,7 @@ fn sample_returns_token_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; context.decode(&batch)?; - let mut sampler = - LlamaSampler::chain_simple([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; + let mut sampler = LlamaSampler::chain([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; let result = sampler.sample(&context, batch.n_tokens() - 1); assert!(result.is_ok()); diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index 42eefb654..33167fdda 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -23,7 +23,6 @@ use crate::llama_backend::LlamaBackend; use crate::llama_batch::LlamaBatch; use crate::model::LlamaModel; use crate::model::lora_adapter_scale::LoraAdapterScale; -use crate::timing::LlamaTimings; use crate::token::LlamaToken; use crate::token::data::LlamaTokenData; use crate::token::data_array::LlamaTokenDataArray; @@ -841,34 +840,14 @@ impl<'model> LlamaContext<'model> { } } - /// # Errors - /// Returns `LogitsError` if logits are null or `n_vocab` overflows. - pub fn candidates(&self) -> Result + '_, LogitsError> { - let logits = self.get_logits()?; - - Ok((0_i32..).zip(logits).map(|(token_id, logit)| { - let token = LlamaToken::new(token_id); - LlamaTokenData::new(token, *logit, 0_f32) - })) - } - - /// # Errors - /// Returns `LogitsError` if logits are null or `n_vocab` overflows. - pub fn token_data_array(&self) -> Result { - Ok(LlamaTokenDataArray::from_iter(self.candidates()?, false)) - } - - fn produced_logits(&self) -> Result, LogitsError> { - NonNull::new(unsafe { llama_cpp_bindings_sys::llama_get_logits(self.context.as_ptr()) }) - .ok_or_else(|| self.decoded_outputs.missing_logits_error()) - } - - /// # Errors - /// Returns `LogitsError` if the last decode produced no logits or `n_vocab` overflows. - pub fn get_logits(&self) -> Result<&[f32], LogitsError> { - let logits = self.produced_logits()?; - - unsafe { logits_slice(logits, self.model.n_vocab()) } + fn ensure_logits_produced(&self) -> Result<(), LogitsError> { + if unsafe { + llama_cpp_bindings_sys::llama_rs_context_produced_logits(self.context.as_ptr()) + } { + Ok(()) + } else { + Err(self.decoded_outputs.missing_logits_error()) + } } /// # Errors @@ -921,7 +900,7 @@ impl<'model> LlamaContext<'model> { /// last decode produced no logits, or `n_vocab` overflows. pub fn get_logits_ith(&self, token_index: i32) -> Result<&[f32], LogitsError> { self.decoded_outputs.ensure_output(token_index)?; - self.produced_logits()?; + self.ensure_logits_produced()?; let row = NonNull::new(unsafe { llama_cpp_bindings_sys::llama_rs_get_logits_ith(self.context.as_ptr(), token_index) @@ -931,16 +910,6 @@ impl<'model> LlamaContext<'model> { unsafe { logits_slice(row, self.model.n_vocab()) } } - pub fn reset_timings(&mut self) { - unsafe { llama_cpp_bindings_sys::llama_perf_context_reset(self.context.as_ptr()) } - } - - pub fn timings(&mut self) -> LlamaTimings { - LlamaTimings::from(unsafe { - llama_cpp_bindings_sys::llama_perf_context(self.context.as_ptr()) - }) - } - /// # Errors /// /// See [`LlamaLoraAdaptersError`] for more information. diff --git a/llama-cpp-bindings/src/lib.rs b/llama-cpp-bindings/src/lib.rs index a9d3e9c9d..a59974b27 100644 --- a/llama-cpp-bindings/src/lib.rs +++ b/llama-cpp-bindings/src/lib.rs @@ -76,7 +76,6 @@ pub mod sampling; pub mod sanitized_grammar; pub mod send_logs_to_log; pub mod streaming_json_probe; -pub mod timing; pub mod token; pub mod token_piece; pub mod tool_call_format; diff --git a/llama-cpp-bindings/src/llama_backend.rs b/llama-cpp-bindings/src/llama_backend.rs index adf6df09c..5ffbb7ed0 100644 --- a/llama-cpp-bindings/src/llama_backend.rs +++ b/llama-cpp-bindings/src/llama_backend.rs @@ -27,16 +27,21 @@ impl LlamaBackend { Ok(Self {}) } + /// Initializes the backend, then llama.cpp's NUMA support, which needs the CPU backend the + /// backend initialization loads. + /// /// # Errors /// Returns an error if the backend was already initialized. pub fn init_numa(strategy: NumaStrategy) -> crate::Result { - Self::mark_init()?; + let backend = Self::init()?; + unsafe { llama_cpp_bindings_sys::llama_numa_init( llama_cpp_bindings_sys::ggml_numa_strategy::from(strategy), ); } - Ok(Self {}) + + Ok(backend) } #[must_use] @@ -148,10 +153,12 @@ mod tests { #[test] #[serial] - fn init_numa_succeeds() { + fn init_numa_with_a_numa_strategy_initializes_the_backend_numa_support_needs() { use crate::llama_backend_numa_strategy::NumaStrategy; - let backend = LlamaBackend::init_numa(NumaStrategy::Disabled); - assert!(backend.is_ok()); + assert_eq!( + LlamaBackend::init_numa(NumaStrategy::Distribute), + Ok(LlamaBackend {}) + ); } } diff --git a/llama-cpp-bindings/src/llama_token_attrs.rs b/llama-cpp-bindings/src/llama_token_attrs.rs index d37d297ca..00c6f3cff 100644 --- a/llama-cpp-bindings/src/llama_token_attrs.rs +++ b/llama-cpp-bindings/src/llama_token_attrs.rs @@ -22,10 +22,10 @@ impl DerefMut for LlamaTokenAttrs { } } -impl TryFrom for LlamaTokenAttrs { +impl TryFrom for LlamaTokenAttrs { type Error = LlamaTokenAttrsFromIntError; - fn try_from(value: llama_cpp_bindings_sys::llama_vocab_type) -> Result { + fn try_from(value: llama_cpp_bindings_sys::llama_token_attr) -> Result { Ok(Self(BitFlags::from_bits(value).map_err( |bit_flag_error| { LlamaTokenAttrsFromIntError::UnknownValue(bit_flag_error.invalid_bits()) diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index 3582d6938..8eda66824 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -306,14 +306,14 @@ impl LlamaModel { /// # Errors /// /// Returns [`TokenAttrError::TokenOutsideVocabulary`] when the token is not part of this - /// model's vocabulary, or [`TokenAttrError::UnknownAttributes`] when the token type is not - /// known to this library. + /// model's vocabulary, or [`TokenAttrError::UnknownAttributes`] when llama.cpp reports + /// attributes this library does not know. pub fn token_attr(&self, token: LlamaToken) -> Result { let LlamaToken(id) = self.require_vocabulary_token(token)?; - let token_type = + let token_attr = unsafe { llama_cpp_bindings_sys::llama_vocab_get_attr(self.vocab_ptr(), id) }; - LlamaTokenAttrs::try_from(token_type).map_err(TokenAttrError::UnknownAttributes) + LlamaTokenAttrs::try_from(token_attr).map_err(TokenAttrError::UnknownAttributes) } /// # Errors diff --git a/llama-cpp-bindings/src/model/params.rs b/llama-cpp-bindings/src/model/params.rs index 3775b7834..25d5ec01b 100644 --- a/llama-cpp-bindings/src/model/params.rs +++ b/llama-cpp-bindings/src/model/params.rs @@ -1,4 +1,4 @@ -use std::ffi::{CStr, c_char}; +use std::ffi::{CStr, CString, c_char}; use std::fmt::{Debug, Formatter}; use std::pin::Pin; use std::ptr::null; @@ -129,6 +129,7 @@ pub struct LlamaModelParams { pub params: llama_cpp_bindings_sys::llama_model_params, kv_overrides: Vec, buft_overrides: Vec, + buft_override_patterns: Vec, devices: Pin>, tensor_split: Vec, } @@ -239,9 +240,12 @@ impl LlamaModelParams { })?; } - buft_override.pattern = key.as_ptr(); + let pattern = key.to_owned(); + + buft_override.pattern = pattern.as_ptr(); buft_override.buft = unsafe { llama_cpp_bindings_sys::ggml_backend_cpu_buffer_type() }; + self.buft_override_patterns.push(pattern); self.push_buft_override_terminator(); Ok(()) @@ -479,6 +483,7 @@ impl Default for LlamaModelParams { pattern: null(), buft: std::ptr::null_mut(), }], + buft_override_patterns: Vec::new(), devices: Box::pin([std::ptr::null_mut(); 16]), tensor_split: Vec::new(), } @@ -724,6 +729,23 @@ mod tests { assert!(result.is_ok()); } + #[test] + fn a_buft_override_pattern_stays_valid_after_the_callers_buffer_changes() { + let mut caller_buffer = b"blk\\.0\\.ffn\0".to_vec(); + let mut params = std::pin::pin!(LlamaModelParams::default()); + + params + .as_mut() + .add_cpu_buft_override(std::ffi::CStr::from_bytes_with_nul(&caller_buffer).unwrap()) + .unwrap(); + caller_buffer.fill(b'x'); + + assert_eq!( + unsafe { std::ffi::CStr::from_ptr(params.buft_overrides[0].pattern) }, + c"blk\\.0\\.ffn" + ); + } + #[test] fn add_cpu_buft_override_twice_fails_with_slot_not_empty() { let mut params = std::pin::pin!(LlamaModelParams::default()); diff --git a/llama-cpp-bindings/src/model/rope_type.rs b/llama-cpp-bindings/src/model/rope_type.rs index cf3173db6..26f4dd44c 100644 --- a/llama-cpp-bindings/src/model/rope_type.rs +++ b/llama-cpp-bindings/src/model/rope_type.rs @@ -3,6 +3,7 @@ pub enum RopeType { Norm, NeoX, MRope, + IMRope, Vision, } @@ -13,6 +14,7 @@ impl RopeType { llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NORM => Some(Self::Norm), llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NEOX => Some(Self::NeoX), llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_MROPE => Some(Self::MRope), + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_IMROPE => Some(Self::IMRope), llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_VISION => Some(Self::Vision), _ => None, } @@ -55,6 +57,14 @@ mod tests { ); } + #[test] + fn rope_type_interleaved_mrope() { + assert_eq!( + RopeType::from_raw(llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_IMROPE), + Some(RopeType::IMRope) + ); + } + #[test] fn rope_type_vision() { assert_eq!( diff --git a/llama-cpp-bindings/src/mtmd.rs b/llama-cpp-bindings/src/mtmd.rs index 38b20d450..1d511b834 100644 --- a/llama-cpp-bindings/src/mtmd.rs +++ b/llama-cpp-bindings/src/mtmd.rs @@ -1,3 +1,4 @@ +mod eval_chunk; mod mtmd_ffi_input_part; pub mod fitted_mtmd_input_chunk; diff --git a/llama-cpp-bindings/src/mtmd/eval_chunk.rs b/llama-cpp-bindings/src/mtmd/eval_chunk.rs new file mode 100644 index 000000000..83e1bd244 --- /dev/null +++ b/llama-cpp-bindings/src/mtmd/eval_chunk.rs @@ -0,0 +1,310 @@ +use std::ffi::c_char; + +use llama_cpp_ffi_status::read_and_free_cpp_string; + +use crate::context::LlamaContext; +use crate::context::decoded_output_rows::DecodedOutputRows; + +use super::mtmd_context::MtmdContext; +use super::mtmd_eval_error::MtmdEvalError; +use super::mtmd_input_chunk::MtmdInputChunk; + +fn eval_chunk_single_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_mtmd_eval_chunk_single_status, + final_position: llama_cpp_bindings_sys::llama_pos, + out_llama_cpp_return_code: i32, + out_error: *mut c_char, +) -> Result { + match status { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_OK => Ok(final_position), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_RETURNED_NONZERO_CODE => { + Err(MtmdEvalError::EvalFailed { + code: out_llama_cpp_return_code, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_ERROR_STRING_ALLOCATION_FAILED => { + Err(MtmdEvalError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_OUT_OF_MEMORY => { + Err(MtmdEvalError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + "llama_rs_mtmd_eval_chunk_single", + "reported a thrown C++ exception without an error message", + ) + }?; + Err(MtmdEvalError::Reported { message }) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_MTMD_CTX_ARG => { + Err(crate::FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null mtmd_ctx argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_LLAMA_CTX_ARG => { + Err(crate::FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null llama_ctx argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_CHUNK_ARG => { + Err(crate::FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null chunk argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_OUT_NEW_N_PAST_ARG => { + Err(crate::FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null out_new_n_past argument", + } + .into()) + } + other => Err(crate::FfiStatusError { + operation: "llama_rs_mtmd_eval_chunk_single", + code: i64::from(other), + } + .into()), + } +} + +/// Encodes `chunk` when it holds media and decodes it into `llama_ctx` from `start_position`, +/// returning the position after it. +pub fn eval_chunk( + chunk: &MtmdInputChunk, + mtmd_ctx: &MtmdContext, + llama_ctx: &mut LlamaContext, + start_position: llama_cpp_bindings_sys::llama_pos, + seq_id: llama_cpp_bindings_sys::llama_seq_id, + n_batch: i32, + logits_last: bool, +) -> Result { + let mut final_position: llama_cpp_bindings_sys::llama_pos = start_position; + let mut out_llama_cpp_return_code: i32 = 0; + let mut out_error: *mut c_char = std::ptr::null_mut(); + + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_mtmd_eval_chunk_single( + mtmd_ctx.context.as_ptr(), + llama_ctx.context.as_ptr(), + chunk.chunk.as_ptr(), + start_position, + seq_id, + n_batch, + logits_last, + &raw mut final_position, + &raw mut out_llama_cpp_return_code, + &raw mut out_error, + ) + }; + + let next_position = eval_chunk_single_status_to_result( + status, + final_position, + out_llama_cpp_return_code, + out_error, + )?; + llama_ctx.record_decoded_rows(DecodedOutputRows::of_helper_decode(logits_last)); + + Ok(next_position) +} + +#[cfg(test)] +mod eval_chunk_single_status_to_result_tests { + use std::ptr; + + use llama_cpp_ffi_status::read_and_free_cpp_string; + use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer; + + use super::eval_chunk_single_status_to_result; + use crate::mtmd::mtmd_eval_error::MtmdEvalError; + + #[test] + fn eval_chunk_single_status_cxx_exception_surfaces_the_reported_message() { + let expected_message = unsafe { + read_and_free_cpp_string( + wrapper_allocated_error_message_pointer(), + "llama_rs_mtmd_eval_chunk_single", + "reported a thrown C++ exception without an error message", + ) + }; + let result = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + 0, + wrapper_allocated_error_message_pointer(), + ); + + assert_eq!( + result.err(), + expected_message + .ok() + .map(|message| MtmdEvalError::Reported { message }) + ); + } + + #[test] + fn eval_chunk_single_status_ok_returns_final_position() { + let result = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_OK, + 7, + 0, + ptr::null_mut(), + ); + + assert_eq!(result, Ok(7)); + } + + #[test] + fn eval_chunk_single_status_nonzero_code_maps_to_eval_failed() { + let result = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_RETURNED_NONZERO_CODE, + 0, + -3, + ptr::null_mut(), + ); + + assert_eq!(result, Err(MtmdEvalError::EvalFailed { code: -3 })); + } + + #[test] + fn eval_chunk_single_status_allocation_failed_maps_to_not_enough_memory() { + let result = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_ERROR_STRING_ALLOCATION_FAILED, + 0, + 0, + ptr::null_mut(), + ); + + assert_eq!(result, Err(MtmdEvalError::NotEnoughMemory)); + } + + #[test] + fn eval_chunk_single_status_cxx_exception_reports_unknown_error_for_null() { + let result = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + 0, + ptr::null_mut(), + ); + + assert_eq!( + result, + Err(crate::FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "reported a thrown C++ exception without an error message", + } + .into()) + ); + } + + #[test] + fn eval_chunk_single_unknown_status_is_preserved() { + let result = eval_chunk_single_status_to_result(255, 0, 0, ptr::null_mut()); + + assert_eq!( + result, + Err(MtmdEvalError::FfiStatus(crate::FfiStatusError { + operation: "llama_rs_mtmd_eval_chunk_single", + code: 255, + })) + ); + } + + #[test] + fn eval_chunk_single_status_to_result_maps_null_mtmd_ctx_arg() { + let outcome = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_MTMD_CTX_ARG, + 0, + 0, + ptr::null_mut(), + ); + assert_eq!( + outcome.err(), + Some( + crate::FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null mtmd_ctx argument", + } + .into() + ) + ); + } + + #[test] + fn eval_chunk_single_status_to_result_maps_null_llama_ctx_arg() { + let outcome = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_LLAMA_CTX_ARG, + 0, + 0, + ptr::null_mut(), + ); + assert_eq!( + outcome.err(), + Some( + crate::FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null llama_ctx argument", + } + .into() + ) + ); + } + + #[test] + fn eval_chunk_single_status_to_result_maps_null_chunk_arg() { + let outcome = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_CHUNK_ARG, + 0, + 0, + ptr::null_mut(), + ); + assert_eq!( + outcome.err(), + Some( + crate::FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null chunk argument", + } + .into() + ) + ); + } + + #[test] + fn eval_chunk_single_status_to_result_maps_null_out_new_n_past_arg() { + let outcome = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_OUT_NEW_N_PAST_ARG, + 0, + 0, + ptr::null_mut(), + ); + assert_eq!( + outcome.err(), + Some( + crate::FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null out_new_n_past argument", + } + .into() + ) + ); + } + + #[test] + fn eval_chunk_single_status_to_result_maps_llama_cpp_out_of_memory() { + let outcome = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_OUT_OF_MEMORY, + 0, + 0, + ptr::null_mut(), + ); + assert_eq!(outcome.err(), Some(MtmdEvalError::LlamaCppOutOfMemory)); + } +} diff --git a/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs b/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs index 6486f7fdc..f56a3324d 100644 --- a/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs @@ -1,11 +1,9 @@ use std::borrow::Borrow; -use std::ffi::c_char; use std::marker::PhantomData; use crate::context::LlamaContext; -use crate::context::decoded_output_rows::DecodedOutputRows; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use super::eval_chunk::eval_chunk; use super::fitted_mtmd_input_chunks::FittedMtmdInputChunks; use super::mtmd_context::MtmdContext; use super::mtmd_eval_error::MtmdEvalError; @@ -14,71 +12,6 @@ use super::mtmd_input_chunk::MtmdInputChunk; use super::mtmd_input_chunk_type::MtmdInputChunkType; use super::mtmd_input_chunks::MtmdInputChunks; -fn eval_chunk_single_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_mtmd_eval_chunk_single_status, - final_position: llama_cpp_bindings_sys::llama_pos, - out_llama_cpp_return_code: i32, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_OK => Ok(final_position), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_RETURNED_NONZERO_CODE => { - Err(MtmdEvalError::EvalFailed { - code: out_llama_cpp_return_code, - }) - } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_ERROR_STRING_ALLOCATION_FAILED => { - Err(MtmdEvalError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(MtmdEvalError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - out_error, - "llama_rs_mtmd_eval_chunk_single", - "reported a thrown C++ exception without an error message", - ) - }?; - Err(MtmdEvalError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_MTMD_CTX_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null mtmd_ctx argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_LLAMA_CTX_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null llama_ctx argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_CHUNK_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null chunk argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_OUT_NEW_N_PAST_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null out_new_n_past argument", - } - .into()) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_mtmd_eval_chunk_single", - code: i64::from(other), - } - .into()), - } -} - #[derive(Debug)] pub struct FittedMtmdInputChunk<'fitted> { chunk: MtmdInputChunk, @@ -143,227 +76,14 @@ impl<'fitted> FittedMtmdInputChunk<'fitted> { }); } - let mut final_position: llama_cpp_bindings_sys::llama_pos = start_position; - let mut out_llama_cpp_return_code: i32 = 0; - let mut out_error: *mut c_char = std::ptr::null_mut(); - - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_mtmd_eval_chunk_single( - mtmd_ctx.context.as_ptr(), - llama_ctx.context.as_ptr(), - self.chunk.chunk.as_ptr(), - start_position, - seq_id, - n_batch, - logits_last, - &raw mut final_position, - &raw mut out_llama_cpp_return_code, - &raw mut out_error, - ) - }; - - let next_position = eval_chunk_single_status_to_result( - status, - final_position, - out_llama_cpp_return_code, - out_error, - )?; - llama_ctx.record_decoded_rows(DecodedOutputRows::of_helper_decode(logits_last)); - - Ok(next_position) - } -} - -#[cfg(test)] -mod eval_chunk_single_status_to_result_tests { - use std::ptr; - - use llama_cpp_ffi_status::read_and_free_cpp_string; - use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer; - - use super::eval_chunk_single_status_to_result; - use crate::mtmd::mtmd_eval_error::MtmdEvalError; - - #[test] - fn eval_chunk_single_status_cxx_exception_surfaces_the_reported_message() { - let expected_message = unsafe { - read_and_free_cpp_string( - wrapper_allocated_error_message_pointer(), - "llama_rs_mtmd_eval_chunk_single", - "reported a thrown C++ exception without an error message", - ) - }; - let result = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_THREW_CXX_EXCEPTION, - 0, - 0, - wrapper_allocated_error_message_pointer(), - ); - - assert_eq!( - result.err(), - expected_message - .ok() - .map(|message| MtmdEvalError::Reported { message }) - ); - } - - #[test] - fn eval_chunk_single_status_ok_returns_final_position() { - let result = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_OK, - 7, - 0, - ptr::null_mut(), - ); - - assert_eq!(result, Ok(7)); - } - - #[test] - fn eval_chunk_single_status_nonzero_code_maps_to_eval_failed() { - let result = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_RETURNED_NONZERO_CODE, - 0, - -3, - ptr::null_mut(), - ); - - assert_eq!(result, Err(MtmdEvalError::EvalFailed { code: -3 })); - } - - #[test] - fn eval_chunk_single_status_allocation_failed_maps_to_not_enough_memory() { - let result = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_ERROR_STRING_ALLOCATION_FAILED, - 0, - 0, - ptr::null_mut(), - ); - - assert_eq!(result, Err(MtmdEvalError::NotEnoughMemory)); - } - - #[test] - fn eval_chunk_single_status_cxx_exception_reports_unknown_error_for_null() { - let result = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_THREW_CXX_EXCEPTION, - 0, - 0, - ptr::null_mut(), - ); - - assert_eq!( - result, - Err(crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "reported a thrown C++ exception without an error message", - } - .into()) - ); - } - - #[test] - fn eval_chunk_single_unknown_status_is_preserved() { - let result = eval_chunk_single_status_to_result(255, 0, 0, ptr::null_mut()); - - assert_eq!( - result, - Err(MtmdEvalError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_mtmd_eval_chunk_single", - code: 255, - })) - ); - } - - #[test] - fn eval_chunk_single_status_to_result_maps_null_mtmd_ctx_arg() { - let outcome = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_MTMD_CTX_ARG, - 0, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null mtmd_ctx argument", - } - .into() - ) - ); - } - - #[test] - fn eval_chunk_single_status_to_result_maps_null_llama_ctx_arg() { - let outcome = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_LLAMA_CTX_ARG, - 0, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null llama_ctx argument", - } - .into() - ) - ); - } - - #[test] - fn eval_chunk_single_status_to_result_maps_null_chunk_arg() { - let outcome = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_CHUNK_ARG, - 0, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null chunk argument", - } - .into() - ) - ); - } - - #[test] - fn eval_chunk_single_status_to_result_maps_null_out_new_n_past_arg() { - let outcome = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_OUT_NEW_N_PAST_ARG, - 0, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null out_new_n_past argument", - } - .into() - ) - ); - } - - #[test] - fn eval_chunk_single_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_OUT_OF_MEMORY, - 0, - 0, - ptr::null_mut(), - ); - assert_eq!(outcome.err(), Some(MtmdEvalError::LlamaCppOutOfMemory)); + eval_chunk( + &self.chunk, + mtmd_ctx, + llama_ctx, + start_position, + seq_id, + n_batch, + logits_last, + ) } } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs index a365f0d7e..cba0be724 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs @@ -5,8 +5,8 @@ use std::ptr::NonNull; use llama_cpp_ffi_status::read_and_free_cpp_string; use crate::context::LlamaContext; -use crate::context::decoded_output_rows::DecodedOutputRows; +use super::eval_chunk::eval_chunk; use super::mtmd_context::MtmdContext; use super::mtmd_eval_error::MtmdEvalError; use super::mtmd_eval_limits::MtmdEvalLimits; @@ -14,14 +14,6 @@ use super::mtmd_input_chunk::MtmdInputChunk; use super::mtmd_input_chunk_type::MtmdInputChunkType; use super::mtmd_input_chunks_error::MtmdInputChunksError; -const fn check_eval_result(result: i32) -> Result<(), MtmdEvalError> { - if result == 0 { - Ok(()) - } else { - Err(MtmdEvalError::EvalFailed { code: result }) - } -} - fn input_chunks_init_status_to_result( status: llama_cpp_bindings_sys::llama_rs_mtmd_input_chunks_init_status, out_chunks: *mut llama_cpp_bindings_sys::mtmd_input_chunks, @@ -137,11 +129,7 @@ impl MtmdInputChunks { ) -> Result, MtmdEvalError> { (0..self.len()) .map(|index| { - self.get(index) - .ok_or(crate::FfiContractError { - operation: "mtmd_input_chunks_get", - detail: "returned a null chunk within the chunk count", - })? + self.chunk_within_count(index)? .fit_to_micro_batch(mtmd_ctx, micro_batch_tokens) }) .collect() @@ -182,25 +170,29 @@ impl MtmdInputChunks { .micro_batch_tokens(limits.batch_tokens(n_batch, seq_id)?), )?; - let mut final_position: llama_cpp_bindings_sys::llama_pos = start_position; + let chunk_count = self.len(); - let result = unsafe { - llama_cpp_bindings_sys::mtmd_helper_eval_chunks( - mtmd_ctx.context.as_ptr(), - llama_ctx.context.as_ptr(), - self.chunks.as_ptr(), - start_position, + (0..chunk_count).try_fold(start_position, |position, index| { + eval_chunk( + &self.chunk_within_count(index)?, + mtmd_ctx, + llama_ctx, + position, seq_id, n_batch, - logits_last, - &raw mut final_position, + logits_last && index + 1 == chunk_count, ) - }; - - check_eval_result(result)?; - llama_ctx.record_decoded_rows(DecodedOutputRows::of_helper_decode(logits_last)); + }) + } - Ok(final_position) + fn chunk_within_count(&self, index: usize) -> Result { + self.get(index).ok_or_else(|| { + crate::FfiContractError { + operation: "mtmd_input_chunks_get", + detail: "returned a null chunk within the chunk count", + } + .into() + }) } } @@ -229,27 +221,6 @@ mod tests { assert!(chunks.get(0).is_none()); assert!(chunks.get(999).is_none()); } - - #[test] - fn check_eval_result_ok_for_zero() { - use super::check_eval_result; - - assert!(check_eval_result(0).is_ok()); - } - - #[test] - fn check_eval_result_error_for_nonzero() { - use super::MtmdEvalError; - use super::check_eval_result; - - let err = check_eval_result(7).unwrap_err(); - let representative = MtmdEvalError::EvalFailed { code: 0 }; - - assert_eq!( - std::mem::discriminant(&err), - std::mem::discriminant(&representative) - ); - } } #[cfg(test)] diff --git a/llama-cpp-bindings/src/sampling.rs b/llama-cpp-bindings/src/sampling.rs index 550688c38..4ac5e99d1 100644 --- a/llama-cpp-bindings/src/sampling.rs +++ b/llama-cpp-bindings/src/sampling.rs @@ -515,13 +515,10 @@ impl LlamaSampler { /// # Errors /// /// Returns [`SamplingError`] if the sampler chain cannot be initialized. - pub fn chain( - samplers: impl IntoIterator, - no_perf: bool, - ) -> Result { + pub fn chain(samplers: impl IntoIterator) -> Result { Self::initialize("chain", |out_sampler, out_error| unsafe { llama_cpp_bindings_sys::llama_rs_sampler_chain_init( - llama_cpp_bindings_sys::llama_sampler_chain_params { no_perf }, + llama_cpp_bindings_sys::llama_sampler_chain_params { no_perf: true }, out_sampler, out_error, ) @@ -534,13 +531,6 @@ impl LlamaSampler { }) } - /// # Errors - /// - /// Returns [`SamplingError`] if the sampler chain cannot be initialized. - pub fn chain_simple(samplers: impl IntoIterator) -> Result { - Self::chain(samplers, false) - } - /// # Errors /// /// Returns [`SamplingError`] if the temperature sampler cannot be initialized. @@ -1138,7 +1128,7 @@ mod tests { #[test] fn accept_succeeds() { - let mut sampler = LlamaSampler::chain_simple([penalties_sampler(), greedy_sampler()]) + let mut sampler = LlamaSampler::chain([penalties_sampler(), greedy_sampler()]) .expect("sampler chain must initialize"); sampler @@ -1148,7 +1138,7 @@ mod tests { #[test] fn accept_succeeds_on_penalties_sampler() { - let mut sampler = LlamaSampler::chain_simple([penalties_sampler(), greedy_sampler()]) + let mut sampler = LlamaSampler::chain([penalties_sampler(), greedy_sampler()]) .expect("sampler chain must initialize"); let result = sampler.accept(crate::token::LlamaToken::new(42)); @@ -1160,7 +1150,7 @@ mod tests { fn accept_many_multiple_tokens() { use crate::token::LlamaToken; - let mut sampler = LlamaSampler::chain_simple([penalties_sampler(), greedy_sampler()]) + let mut sampler = LlamaSampler::chain([penalties_sampler(), greedy_sampler()]) .expect("sampler chain must initialize"); sampler @@ -1172,7 +1162,7 @@ mod tests { fn with_tokens_builder_pattern() { use crate::token::LlamaToken; - let _sampler = LlamaSampler::chain_simple([penalties_sampler(), greedy_sampler()]) + let _sampler = LlamaSampler::chain([penalties_sampler(), greedy_sampler()]) .expect("sampler chain must initialize") .with_tokens([LlamaToken::new(10), LlamaToken::new(20)]) .expect("test: with_tokens should succeed"); @@ -1201,7 +1191,7 @@ mod tests { let biases = vec![LlamaLogitBias::new(LlamaToken::new(0), -100.0)]; let _logit_bias = LlamaSampler::logit_bias(32000, &biases); let _chain = - LlamaSampler::chain([greedy_sampler()], true).expect("sampler chain must initialize"); + LlamaSampler::chain([greedy_sampler()]).expect("sampler chain must initialize"); } #[test] diff --git a/llama-cpp-bindings/src/timing.rs b/llama-cpp-bindings/src/timing.rs deleted file mode 100644 index d42adaedf..000000000 --- a/llama-cpp-bindings/src/timing.rs +++ /dev/null @@ -1,210 +0,0 @@ -use std::fmt::{Display, Formatter, Write}; - -use llama_cpp_bindings_sys::llama_perf_context_data; - -fn write_timings(timings: &LlamaTimings, writer: &mut dyn Write) -> std::fmt::Result { - writeln!(writer, "load time = {:.2} ms", timings.t_load_ms())?; - - if timings.n_p_eval() > 0 { - writeln!( - writer, - "prompt eval time = {:.2} ms / {} tokens ({:.2} ms per token, {:.2} tokens per second)", - timings.t_p_eval_ms(), - timings.n_p_eval(), - timings.t_p_eval_ms() / f64::from(timings.n_p_eval()), - 1e3 / timings.t_p_eval_ms() * f64::from(timings.n_p_eval()) - )?; - } else { - writeln!( - writer, - "prompt eval time = {:.2} ms / 0 tokens", - timings.t_p_eval_ms(), - )?; - } - - if timings.n_eval() > 0 { - writeln!( - writer, - "eval time = {:.2} ms / {} runs ({:.2} ms per token, {:.2} tokens per second)", - timings.t_eval_ms(), - timings.n_eval(), - timings.t_eval_ms() / f64::from(timings.n_eval()), - 1e3 / timings.t_eval_ms() * f64::from(timings.n_eval()) - )?; - } else { - writeln!(writer, "eval time = {:.2} ms / 0 runs", timings.t_eval_ms())?; - } - - writeln!(writer, "graphs reused = {}", timings.n_reused())?; - writeln!( - writer, - "graph reallocs = {} with the same size / {} with a changed size", - timings.n_reallocs_same(), - timings.n_reallocs_diff() - )?; - - Ok(()) -} - -#[derive(Clone, Copy, Debug)] -pub struct LlamaTimings { - timings: llama_perf_context_data, -} - -impl From for LlamaTimings { - fn from(timings: llama_perf_context_data) -> Self { - Self { timings } - } -} - -impl LlamaTimings { - #[must_use] - pub const fn t_start_ms(&self) -> f64 { - self.timings.t_start_ms - } - - #[must_use] - pub const fn t_load_ms(&self) -> f64 { - self.timings.t_load_ms - } - - #[must_use] - pub const fn t_p_eval_ms(&self) -> f64 { - self.timings.t_p_eval_ms - } - - #[must_use] - pub const fn t_eval_ms(&self) -> f64 { - self.timings.t_eval_ms - } - - #[must_use] - pub const fn n_p_eval(&self) -> i32 { - self.timings.n_p_eval - } - - #[must_use] - pub const fn n_eval(&self) -> i32 { - self.timings.n_eval - } - - #[must_use] - pub const fn n_reused(&self) -> i32 { - self.timings.n_reused - } - - #[must_use] - pub const fn n_reallocs_same(&self) -> i32 { - self.timings.n_reallocs_same - } - - #[must_use] - pub const fn n_reallocs_diff(&self) -> i32 { - self.timings.n_reallocs_diff - } -} - -impl Display for LlamaTimings { - fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { - write_timings(self, formatter) - } -} - -#[cfg(test)] -mod tests { - use std::fmt::Write; - - use llama_cpp_bindings_sys::llama_perf_context_data; - - use super::LlamaTimings; - use super::write_timings; - - struct WriterFailingAfterWrites { - remaining_writes: usize, - } - - impl Write for WriterFailingAfterWrites { - fn write_str(&mut self, _text: &str) -> std::fmt::Result { - if self.remaining_writes == 0 { - return Err(std::fmt::Error); - } - self.remaining_writes -= 1; - - Ok(()) - } - } - - struct WriterCountingWrites { - writes: usize, - } - - impl Write for WriterCountingWrites { - fn write_str(&mut self, _text: &str) -> std::fmt::Result { - self.writes += 1; - - Ok(()) - } - } - - const fn timings_with_eval_counts(n_p_eval: i32, n_eval: i32) -> LlamaTimings { - LlamaTimings { - timings: llama_perf_context_data { - t_start_ms: 1.0, - t_load_ms: 2.0, - t_p_eval_ms: 3.0, - t_eval_ms: 4.0, - n_p_eval, - n_eval, - n_reused: 7, - n_reallocs_same: 8, - n_reallocs_diff: 9, - }, - } - } - - #[test] - fn display_reports_throughput_of_evaluated_prompt_and_generated_tokens() { - let output = timings_with_eval_counts(5, 6).to_string(); - - assert_eq!( - output, - "load time = 2.00 ms\n\ - prompt eval time = 3.00 ms / 5 tokens (0.60 ms per token, 1666.67 tokens per second)\n\ - eval time = 4.00 ms / 6 runs (0.67 ms per token, 1500.00 tokens per second)\n\ - graphs reused = 7\n\ - graph reallocs = 8 with the same size / 9 with a changed size\n" - ); - } - - #[test] - fn display_omits_throughput_when_nothing_was_evaluated() { - let output = timings_with_eval_counts(0, 0).to_string(); - - assert_eq!( - output, - "load time = 2.00 ms\n\ - prompt eval time = 3.00 ms / 0 tokens\n\ - eval time = 4.00 ms / 0 runs\n\ - graphs reused = 7\n\ - graph reallocs = 8 with the same size / 9 with a changed size\n" - ); - } - - #[test] - fn write_timings_propagates_a_writer_failure_at_every_write() { - for timings in [ - timings_with_eval_counts(5, 6), - timings_with_eval_counts(0, 0), - ] { - let mut counting_writer = WriterCountingWrites { writes: 0 }; - assert_eq!(write_timings(&timings, &mut counting_writer), Ok(())); - - for remaining_writes in 0..counting_writer.writes { - assert_eq!( - write_timings(&timings, &mut WriterFailingAfterWrites { remaining_writes }), - Err(std::fmt::Error) - ); - } - } - } -} diff --git a/llama-cpp-bindings/tests/sampler_chain_without_backend.rs b/llama-cpp-bindings/tests/sampler_chain_without_backend.rs index f8ff4ee0c..c9175447c 100644 --- a/llama-cpp-bindings/tests/sampler_chain_without_backend.rs +++ b/llama-cpp-bindings/tests/sampler_chain_without_backend.rs @@ -6,7 +6,7 @@ use llama_cpp_bindings::token::data_array::LlamaTokenDataArray; #[test] fn sampler_chain_applies_without_initialized_backend() -> Result<()> { - let sampler_chain = LlamaSampler::chain_simple([LlamaSampler::greedy()?])?; + let sampler_chain = LlamaSampler::chain([LlamaSampler::greedy()?])?; let mut candidates = LlamaTokenDataArray::new( vec![ LlamaTokenData::new(LlamaToken::new(0), 1.0, 0.0), From 790840245617888f4916cb9a3a78b1940e64977b Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sat, 31 Oct 2026 04:46:26 +0100 Subject: [PATCH 22/82] Apply samplers, log, time and query memory one way each, and name memory and state operations as llama.cpp does --- llama-cpp-bindings-tests/src/lib.rs | 4 +- .../src/prime_kv_cache.rs | 11 - llama-cpp-bindings-tests/src/prime_memory.rs | 11 + ..._kv_cache_with.rs => prime_memory_with.rs} | 2 +- .../tests/embedding_generation.rs | 4 +- .../tests/hidden_state_extraction.rs | 4 +- llama-cpp-bindings-tests/tests/main.rs | 14 +- ...availability.rs => memory_availability.rs} | 38 +- ...cache_occupancy.rs => memory_occupancy.rs} | 24 +- ...on_bounds.rs => memory_position_bounds.rs} | 56 +- ...mory_position_shift_rope_compatibility.rs} | 40 +- ...{kv_cache_priming.rs => memory_priming.rs} | 12 +- ...ence_edits.rs => memory_sequence_edits.rs} | 31 +- ...bounds.rs => memory_sequence_id_bounds.rs} | 16 +- .../tests/multimodal_chunk_evaluation.rs | 26 +- llama-cpp-bindings-tests/tests/reranking.rs | 2 +- .../tests/sampler_token_acceptance.rs | 26 - .../tests/state_snapshots.rs | 20 +- .../src/backend_device_index.rs | 12 + llama-cpp-bindings/src/context.rs | 5 +- .../src/context/{kv_cache.rs => memory.rs} | 259 +++++----- llama-cpp-bindings/src/context/session.rs | 8 +- llama-cpp-bindings/src/error.rs | 22 +- ...q_div_error.rs => memory_seq_add_error.rs} | 2 +- ...he_seq_error.rs => memory_seq_cp_error.rs} | 2 +- ...q_add_error.rs => memory_seq_div_error.rs} | 2 +- ...x_error.rs => memory_seq_pos_max_error.rs} | 2 +- ...he_seq_error.rs => memory_seq_rm_error.rs} | 2 +- .../src/error/token_sampling_error.rs | 12 - llama-cpp-bindings/src/ggml_time_us.rs | 21 - llama-cpp-bindings/src/lib.rs | 21 +- llama-cpp-bindings/src/llama_backend.rs | 26 - llama-cpp-bindings/src/llguidance_sampler.rs | 13 - llama-cpp-bindings/src/mlock_supported.rs | 14 - llama-cpp-bindings/src/mmap_supported.rs | 14 - llama-cpp-bindings/src/model/params.rs | 9 +- .../src/mtmd/mtmd_memory_usage.rs | 12 +- llama-cpp-bindings/src/sampling.rs | 315 ++++++++++- llama-cpp-bindings/src/token/data_array.rs | 488 +----------------- .../tests/sampler_chain_without_backend.rs | 2 +- .../src/run_to_conclusions.rs | 7 +- 41 files changed, 630 insertions(+), 981 deletions(-) delete mode 100644 llama-cpp-bindings-tests/src/prime_kv_cache.rs create mode 100644 llama-cpp-bindings-tests/src/prime_memory.rs rename llama-cpp-bindings-tests/src/{prime_kv_cache_with.rs => prime_memory_with.rs} (96%) rename llama-cpp-bindings-tests/tests/{kv_cache_memory_availability.rs => memory_availability.rs} (60%) rename llama-cpp-bindings-tests/tests/{kv_cache_occupancy.rs => memory_occupancy.rs} (81%) rename llama-cpp-bindings-tests/tests/{kv_cache_position_bounds.rs => memory_position_bounds.rs} (82%) rename llama-cpp-bindings-tests/tests/{kv_cache_position_shift_rope_compatibility.rs => memory_position_shift_rope_compatibility.rs} (63%) rename llama-cpp-bindings-tests/tests/{kv_cache_priming.rs => memory_priming.rs} (71%) rename llama-cpp-bindings-tests/tests/{kv_cache_sequence_edits.rs => memory_sequence_edits.rs} (78%) rename llama-cpp-bindings-tests/tests/{kv_cache_sequence_id_bounds.rs => memory_sequence_id_bounds.rs} (80%) create mode 100644 llama-cpp-bindings/src/backend_device_index.rs rename llama-cpp-bindings/src/context/{kv_cache.rs => memory.rs} (72%) rename llama-cpp-bindings/src/error/{kv_cache_seq_div_error.rs => memory_seq_add_error.rs} (96%) rename llama-cpp-bindings/src/error/{copy_kv_cache_seq_error.rs => memory_seq_cp_error.rs} (92%) rename llama-cpp-bindings/src/error/{kv_cache_seq_add_error.rs => memory_seq_div_error.rs} (96%) rename llama-cpp-bindings/src/error/{kv_cache_seq_pos_max_error.rs => memory_seq_pos_max_error.rs} (94%) rename llama-cpp-bindings/src/error/{clear_kv_cache_seq_error.rs => memory_seq_rm_error.rs} (94%) delete mode 100644 llama-cpp-bindings/src/error/token_sampling_error.rs delete mode 100644 llama-cpp-bindings/src/ggml_time_us.rs delete mode 100644 llama-cpp-bindings/src/mlock_supported.rs delete mode 100644 llama-cpp-bindings/src/mmap_supported.rs diff --git a/llama-cpp-bindings-tests/src/lib.rs b/llama-cpp-bindings-tests/src/lib.rs index fc21c709a..2ddc3445c 100644 --- a/llama-cpp-bindings-tests/src/lib.rs +++ b/llama-cpp-bindings-tests/src/lib.rs @@ -13,8 +13,8 @@ pub mod media_question_chat_session; pub mod normalize; pub mod paris_weather_tool_call; pub mod penalized_nucleus_sampler; -pub mod prime_kv_cache; -pub mod prime_kv_cache_with; +pub mod prime_memory; +pub mod prime_memory_with; pub mod prompt_question; pub mod recognized_answer; pub mod tokenize_prompt_with_synthetic_image; diff --git a/llama-cpp-bindings-tests/src/prime_kv_cache.rs b/llama-cpp-bindings-tests/src/prime_kv_cache.rs deleted file mode 100644 index 35fd864a5..000000000 --- a/llama-cpp-bindings-tests/src/prime_kv_cache.rs +++ /dev/null @@ -1,11 +0,0 @@ -use anyhow::Result; -use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_test_harness::LlamaFixture; - -use crate::prime_kv_cache_with::prime_kv_cache_with; - -/// # Errors -/// Forwards tokenization and [`LlamaContext::decode`] errors verbatim. -pub fn prime_kv_cache(fixture: &LlamaFixture<'_>, context: &mut LlamaContext<'_>) -> Result<()> { - prime_kv_cache_with(fixture, context, "Hello world") -} diff --git a/llama-cpp-bindings-tests/src/prime_memory.rs b/llama-cpp-bindings-tests/src/prime_memory.rs new file mode 100644 index 000000000..4704a334e --- /dev/null +++ b/llama-cpp-bindings-tests/src/prime_memory.rs @@ -0,0 +1,11 @@ +use anyhow::Result; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_test_harness::LlamaFixture; + +use crate::prime_memory_with::prime_memory_with; + +/// # Errors +/// Forwards tokenization and [`LlamaContext::decode`] errors verbatim. +pub fn prime_memory(fixture: &LlamaFixture<'_>, context: &mut LlamaContext<'_>) -> Result<()> { + prime_memory_with(fixture, context, "Hello world") +} diff --git a/llama-cpp-bindings-tests/src/prime_kv_cache_with.rs b/llama-cpp-bindings-tests/src/prime_memory_with.rs similarity index 96% rename from llama-cpp-bindings-tests/src/prime_kv_cache_with.rs rename to llama-cpp-bindings-tests/src/prime_memory_with.rs index 38b7ec2e4..7893d47e9 100644 --- a/llama-cpp-bindings-tests/src/prime_kv_cache_with.rs +++ b/llama-cpp-bindings-tests/src/prime_memory_with.rs @@ -8,7 +8,7 @@ use llama_cpp_test_harness::LlamaFixture; /// # Errors /// Forwards tokenization, batch assembly, and [`LlamaContext::decode`] errors verbatim. -pub fn prime_kv_cache_with( +pub fn prime_memory_with( fixture: &LlamaFixture<'_>, context: &mut LlamaContext<'_>, text: &str, diff --git a/llama-cpp-bindings-tests/tests/embedding_generation.rs b/llama-cpp-bindings-tests/tests/embedding_generation.rs index c119b5463..9a57c24e2 100644 --- a/llama-cpp-bindings-tests/tests/embedding_generation.rs +++ b/llama-cpp-bindings-tests/tests/embedding_generation.rs @@ -47,7 +47,7 @@ fn embedding_generation_produces_vectors(fixture: &LlamaFixture<'_>) -> Result<( assert_eq!(classifier.pending_prompt_tokens(), prompt_token_count); assert_eq!(classifier.usage().prompt_tokens, 0); - ctx.clear_kv_cache(); + ctx.memory_clear(); ctx.decode(&batch) .with_context(|| "llama_decode() failed")?; @@ -250,7 +250,7 @@ fn embeddings_returns_distinct_values_when_reused_batch_has_extra_capacity( batch.add_sequence(&tokens, sequence_id, true)?; } - context.clear_kv_cache(); + context.memory_clear(); context.decode(&batch)?; for sequence_index in 0..iteration_inputs.len() { diff --git a/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs b/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs index bbc89a34d..7b7623c4d 100644 --- a/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs +++ b/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs @@ -261,7 +261,7 @@ fn a_released_fork_leaves_its_shared_state_as_an_independent_decode_would( add_tokens(&mut state_batch, &state, 0, 0, false)?; context.decode(&state_batch)?; - context.copy_kv_cache_seq(0, 1, None, None)?; + context.memory_seq_cp(0, 1, None, None)?; let mut first_branch_batch = LlamaBatch::new(ContextLimits::of(&context)); @@ -278,7 +278,7 @@ fn a_released_fork_leaves_its_shared_state_as_an_independent_decode_would( .nextn_embeddings_ith(first_branch_batch.n_tokens() - 1)? .to_vec(); - context.clear_kv_cache_seq(Some(1), None, None)?; + context.memory_seq_rm(Some(1), None, None)?; let mut second_branch_batch = LlamaBatch::new(ContextLimits::of(&context)); diff --git a/llama-cpp-bindings-tests/tests/main.rs b/llama-cpp-bindings-tests/tests/main.rs index 47f0868e8..7ca043e98 100644 --- a/llama-cpp-bindings-tests/tests/main.rs +++ b/llama-cpp-bindings-tests/tests/main.rs @@ -29,19 +29,19 @@ mod grammar_sampler; mod hidden_state_extraction; mod image_question_answering; mod json_schema_constrained_generation; -mod kv_cache_memory_availability; -mod kv_cache_occupancy; -mod kv_cache_position_bounds; -mod kv_cache_position_shift_rope_compatibility; -mod kv_cache_priming; -mod kv_cache_sequence_edits; -mod kv_cache_sequence_id_bounds; mod lazy_grammar_sampler; mod llguidance_sampler; mod logit_bias_sampler; mod logits_availability; mod logits_retrieval; mod lora_adapters; +mod memory_availability; +mod memory_occupancy; +mod memory_position_bounds; +mod memory_position_shift_rope_compatibility; +mod memory_priming; +mod memory_sequence_edits; +mod memory_sequence_id_bounds; mod model_architecture; mod model_derived_data_caching; mod model_hyperparameters; diff --git a/llama-cpp-bindings-tests/tests/kv_cache_memory_availability.rs b/llama-cpp-bindings-tests/tests/memory_availability.rs similarity index 60% rename from llama-cpp-bindings-tests/tests/kv_cache_memory_availability.rs rename to llama-cpp-bindings-tests/tests/memory_availability.rs index 62607c5e3..36282336d 100644 --- a/llama-cpp-bindings-tests/tests/kv_cache_memory_availability.rs +++ b/llama-cpp-bindings-tests/tests/memory_availability.rs @@ -2,11 +2,11 @@ use std::num::NonZeroU8; use anyhow::Context as _; use anyhow::Result; -use llama_cpp_bindings::ClearKvCacheSeqError; -use llama_cpp_bindings::CopyKvCacheSeqError; -use llama_cpp_bindings::KvCacheSeqAddError; -use llama_cpp_bindings::KvCacheSeqDivError; -use llama_cpp_bindings::KvCacheSeqPosMaxError; +use llama_cpp_bindings::MemorySeqAddError; +use llama_cpp_bindings::MemorySeqCpError; +use llama_cpp_bindings::MemorySeqDivError; +use llama_cpp_bindings::MemorySeqPosMaxError; +use llama_cpp_bindings::MemorySeqRmError; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -50,35 +50,35 @@ fn a_causal_embedding_context_keeps_memory(fixture: &LlamaFixture<'_>) -> Result n_threads_batch = 8, embeddings = true, )] -fn kv_cache_operations_respect_an_embedding_context_without_memory( +fn memory_operations_respect_an_embedding_context_without_memory( fixture: &LlamaFixture<'_>, ) -> Result<()> { let mut context = fixture.build_context()?; - context.clear_kv_cache(); - assert_eq!(context.clear_kv_cache_seq(Some(0), None, None), Ok(())); - context.kv_cache_seq_keep(0); + context.memory_clear(); + assert_eq!(context.memory_seq_rm(Some(0), None, None), Ok(())); + context.memory_seq_keep(0); assert_eq!( - context.copy_kv_cache_seq(0, 1, None, None), - Err(CopyKvCacheSeqError::MemoryHandleUnavailable) + context.memory_seq_cp(0, 1, None, None), + Err(MemorySeqCpError::MemoryHandleUnavailable) ); assert_eq!( - context.kv_cache_seq_add(0, None, None, 1), - Err(KvCacheSeqAddError::MemoryHandleUnavailable) + context.memory_seq_add(0, None, None, 1), + Err(MemorySeqAddError::MemoryHandleUnavailable) ); let divisor = NonZeroU8::new(2).context("2 is non-zero")?; assert_eq!( - context.kv_cache_seq_div(0, None, None, divisor), - Err(KvCacheSeqDivError::MemoryHandleUnavailable) + context.memory_seq_div(0, None, None, divisor), + Err(MemorySeqDivError::MemoryHandleUnavailable) ); assert_eq!( - context.kv_cache_seq_pos_max(0), - Err(KvCacheSeqPosMaxError::MemoryHandleUnavailable) + context.memory_seq_pos_max(0), + Err(MemorySeqPosMaxError::MemoryHandleUnavailable) ); assert!(matches!( - context.clear_kv_cache_seq(Some(u32::MAX), None, None), - Err(ClearKvCacheSeqError::SeqIdTooLarge(_)) + context.memory_seq_rm(Some(u32::MAX), None, None), + Err(MemorySeqRmError::SeqIdTooLarge(_)) )); Ok(()) diff --git a/llama-cpp-bindings-tests/tests/kv_cache_occupancy.rs b/llama-cpp-bindings-tests/tests/memory_occupancy.rs similarity index 81% rename from llama-cpp-bindings-tests/tests/kv_cache_occupancy.rs rename to llama-cpp-bindings-tests/tests/memory_occupancy.rs index 1d68ce287..cb8964d95 100644 --- a/llama-cpp-bindings-tests/tests/kv_cache_occupancy.rs +++ b/llama-cpp-bindings-tests/tests/memory_occupancy.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use llama_cpp_bindings_tests::prime_kv_cache::prime_kv_cache; +use llama_cpp_bindings_tests::prime_memory::prime_memory; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -35,13 +35,13 @@ use llama_cpp_test_harness::llama_test; n_batch = 512, n_ubatch = 128, )] -fn clear_kv_cache_resets_positions(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_clear_resets_positions(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; - prime_kv_cache(fixture, &mut context)?; + prime_memory(fixture, &mut context)?; - context.clear_kv_cache(); - assert_eq!(context.kv_cache_seq_pos_max(0)?, -1); + context.memory_clear(); + assert_eq!(context.memory_seq_pos_max(0)?, -1); Ok(()) } @@ -78,12 +78,12 @@ fn clear_kv_cache_resets_positions(fixture: &LlamaFixture<'_>) -> Result<()> { n_batch = 512, n_ubatch = 128, )] -fn kv_cache_seq_pos_max_is_non_negative_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_pos_max_is_non_negative_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; - prime_kv_cache(fixture, &mut context)?; + prime_memory(fixture, &mut context)?; - assert!(context.kv_cache_seq_pos_max(0)? >= 0); + assert!(context.memory_seq_pos_max(0)? >= 0); Ok(()) } @@ -120,14 +120,14 @@ fn kv_cache_seq_pos_max_is_non_negative_after_decode(fixture: &LlamaFixture<'_>) n_batch = 512, n_ubatch = 128, )] -fn kv_cache_seq_keep_retains_specified_sequence(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_keep_retains_specified_sequence(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; - prime_kv_cache(fixture, &mut context)?; + prime_memory(fixture, &mut context)?; - context.kv_cache_seq_keep(0); + context.memory_seq_keep(0); - assert!(context.kv_cache_seq_pos_max(0)? >= 0); + assert!(context.memory_seq_pos_max(0)? >= 0); Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/kv_cache_position_bounds.rs b/llama-cpp-bindings-tests/tests/memory_position_bounds.rs similarity index 82% rename from llama-cpp-bindings-tests/tests/kv_cache_position_bounds.rs rename to llama-cpp-bindings-tests/tests/memory_position_bounds.rs index f3de9b6f1..fd46f7c46 100644 --- a/llama-cpp-bindings-tests/tests/kv_cache_position_bounds.rs +++ b/llama-cpp-bindings-tests/tests/memory_position_bounds.rs @@ -2,10 +2,10 @@ use std::num::NonZeroU8; use anyhow::Context as _; use anyhow::Result; -use llama_cpp_bindings::error::ClearKvCacheSeqError; -use llama_cpp_bindings::error::CopyKvCacheSeqError; -use llama_cpp_bindings::error::KvCacheSeqAddError; -use llama_cpp_bindings::error::KvCacheSeqDivError; +use llama_cpp_bindings::error::MemorySeqAddError; +use llama_cpp_bindings::error::MemorySeqCpError; +use llama_cpp_bindings::error::MemorySeqDivError; +use llama_cpp_bindings::error::MemorySeqRmError; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -41,14 +41,14 @@ use llama_cpp_test_harness::llama_test; n_batch = 512, n_ubatch = 128, )] -fn copy_kv_cache_seq_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_cp_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; - let result = context.copy_kv_cache_seq(0, 1, Some(u32::MAX), None); + let result = context.memory_seq_cp(0, 1, Some(u32::MAX), None); assert_eq!( result.unwrap_err(), - CopyKvCacheSeqError::P0TooLarge( + MemorySeqCpError::P0TooLarge( i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") ) ); @@ -88,14 +88,14 @@ fn copy_kv_cache_seq_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> n_batch = 512, n_ubatch = 128, )] -fn copy_kv_cache_seq_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_cp_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; - let result = context.copy_kv_cache_seq(0, 1, Some(0), Some(u32::MAX)); + let result = context.memory_seq_cp(0, 1, Some(0), Some(u32::MAX)); assert_eq!( result.unwrap_err(), - CopyKvCacheSeqError::P1TooLarge( + MemorySeqCpError::P1TooLarge( i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") ) ); @@ -135,14 +135,14 @@ fn copy_kv_cache_seq_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> n_batch = 512, n_ubatch = 128, )] -fn clear_kv_cache_seq_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_rm_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; - let result = context.clear_kv_cache_seq(Some(0), Some(u32::MAX), None); + let result = context.memory_seq_rm(Some(0), Some(u32::MAX), None); assert_eq!( result.unwrap_err(), - ClearKvCacheSeqError::P0TooLarge( + MemorySeqRmError::P0TooLarge( i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") ) ); @@ -182,14 +182,14 @@ fn clear_kv_cache_seq_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) - n_batch = 512, n_ubatch = 128, )] -fn clear_kv_cache_seq_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_rm_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; - let result = context.clear_kv_cache_seq(Some(0), Some(0), Some(u32::MAX)); + let result = context.memory_seq_rm(Some(0), Some(0), Some(u32::MAX)); assert_eq!( result.unwrap_err(), - ClearKvCacheSeqError::P1TooLarge( + MemorySeqRmError::P1TooLarge( i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") ) ); @@ -229,14 +229,14 @@ fn clear_kv_cache_seq_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) - n_batch = 512, n_ubatch = 128, )] -fn kv_cache_seq_add_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_add_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; - let result = context.kv_cache_seq_add(0, Some(u32::MAX), None, 1); + let result = context.memory_seq_add(0, Some(u32::MAX), None, 1); assert_eq!( result.unwrap_err(), - KvCacheSeqAddError::P0TooLarge( + MemorySeqAddError::P0TooLarge( i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") ) ); @@ -276,14 +276,14 @@ fn kv_cache_seq_add_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> n_batch = 512, n_ubatch = 128, )] -fn kv_cache_seq_add_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_add_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; - let result = context.kv_cache_seq_add(0, Some(0), Some(u32::MAX), 1); + let result = context.memory_seq_add(0, Some(0), Some(u32::MAX), 1); assert_eq!( result.unwrap_err(), - KvCacheSeqAddError::P1TooLarge( + MemorySeqAddError::P1TooLarge( i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") ) ); @@ -323,15 +323,15 @@ fn kv_cache_seq_add_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> n_batch = 512, n_ubatch = 128, )] -fn kv_cache_seq_div_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_div_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; let divisor = NonZeroU8::new(2).context("2 is non-zero")?; - let result = context.kv_cache_seq_div(0, Some(u32::MAX), None, divisor); + let result = context.memory_seq_div(0, Some(u32::MAX), None, divisor); assert_eq!( result.unwrap_err(), - KvCacheSeqDivError::P0TooLarge( + MemorySeqDivError::P0TooLarge( i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") ) ); @@ -371,15 +371,15 @@ fn kv_cache_seq_div_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> n_batch = 512, n_ubatch = 128, )] -fn kv_cache_seq_div_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_div_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; let divisor = NonZeroU8::new(2).context("2 is non-zero")?; - let result = context.kv_cache_seq_div(0, Some(0), Some(u32::MAX), divisor); + let result = context.memory_seq_div(0, Some(0), Some(u32::MAX), divisor); assert_eq!( result.unwrap_err(), - KvCacheSeqDivError::P1TooLarge( + MemorySeqDivError::P1TooLarge( i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") ) ); diff --git a/llama-cpp-bindings-tests/tests/kv_cache_position_shift_rope_compatibility.rs b/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs similarity index 63% rename from llama-cpp-bindings-tests/tests/kv_cache_position_shift_rope_compatibility.rs rename to llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs index 631bf7f29..ad5b3b467 100644 --- a/llama-cpp-bindings-tests/tests/kv_cache_position_shift_rope_compatibility.rs +++ b/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs @@ -2,9 +2,9 @@ use std::num::NonZeroU8; use anyhow::Context as _; use anyhow::Result; -use llama_cpp_bindings::error::KvCacheSeqAddError; -use llama_cpp_bindings::error::KvCacheSeqDivError; -use llama_cpp_bindings_tests::prime_kv_cache::prime_kv_cache; +use llama_cpp_bindings::error::MemorySeqAddError; +use llama_cpp_bindings::error::MemorySeqDivError; +use llama_cpp_bindings_tests::prime_memory::prime_memory; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -24,17 +24,14 @@ use llama_cpp_test_harness::llama_test; n_batch = 512, n_ubatch = 128, )] -fn kv_cache_seq_add_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_add_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; - prime_kv_cache(fixture, &mut context)?; + prime_memory(fixture, &mut context)?; - let result = context.kv_cache_seq_add(0, Some(0), None, 1); + let result = context.memory_seq_add(0, Some(0), None, 1); - assert_eq!( - result.unwrap_err(), - KvCacheSeqAddError::IncompatibleRopeType - ); + assert_eq!(result.unwrap_err(), MemorySeqAddError::IncompatibleRopeType); Ok(()) } @@ -55,18 +52,15 @@ fn kv_cache_seq_add_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> n_batch = 512, n_ubatch = 128, )] -fn kv_cache_seq_div_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_div_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; - prime_kv_cache(fixture, &mut context)?; + prime_memory(fixture, &mut context)?; let divisor = NonZeroU8::new(2).context("2 is non-zero")?; - let result = context.kv_cache_seq_div(0, Some(0), None, divisor); + let result = context.memory_seq_div(0, Some(0), None, divisor); - assert_eq!( - result.unwrap_err(), - KvCacheSeqDivError::IncompatibleRopeType - ); + assert_eq!(result.unwrap_err(), MemorySeqDivError::IncompatibleRopeType); Ok(()) } @@ -79,12 +73,12 @@ fn kv_cache_seq_div_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> n_batch = 512, n_ubatch = 128, )] -fn kv_cache_seq_add_succeeds_on_embedding_model(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_add_succeeds_on_embedding_model(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; - prime_kv_cache(fixture, &mut context)?; + prime_memory(fixture, &mut context)?; - let result = context.kv_cache_seq_add(0, Some(0), None, 1); + let result = context.memory_seq_add(0, Some(0), None, 1); assert!(result.is_ok()); @@ -99,13 +93,13 @@ fn kv_cache_seq_add_succeeds_on_embedding_model(fixture: &LlamaFixture<'_>) -> R n_batch = 512, n_ubatch = 128, )] -fn kv_cache_seq_div_succeeds_on_embedding_model(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_div_succeeds_on_embedding_model(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; - prime_kv_cache(fixture, &mut context)?; + prime_memory(fixture, &mut context)?; let divisor = NonZeroU8::new(2).context("2 is non-zero")?; - let result = context.kv_cache_seq_div(0, Some(0), None, divisor); + let result = context.memory_seq_div(0, Some(0), None, divisor); assert!(result.is_ok()); diff --git a/llama-cpp-bindings-tests/tests/kv_cache_priming.rs b/llama-cpp-bindings-tests/tests/memory_priming.rs similarity index 71% rename from llama-cpp-bindings-tests/tests/kv_cache_priming.rs rename to llama-cpp-bindings-tests/tests/memory_priming.rs index a973a21e4..071e224f5 100644 --- a/llama-cpp-bindings-tests/tests/kv_cache_priming.rs +++ b/llama-cpp-bindings-tests/tests/memory_priming.rs @@ -1,7 +1,7 @@ use anyhow::Result; use llama_cpp_bindings::StringToTokenError; use llama_cpp_bindings::batch_add_error::BatchAddError; -use llama_cpp_bindings_tests::prime_kv_cache_with::prime_kv_cache_with; +use llama_cpp_bindings_tests::prime_memory_with::prime_memory_with; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -13,16 +13,16 @@ use llama_cpp_test_harness::llama_test; n_batch = 256, n_ubatch = 64, )] -fn prime_kv_cache_surfaces_each_underlying_error(fixture: &LlamaFixture<'_>) -> Result<()> { +fn prime_memory_surfaces_each_underlying_error(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; assert!( - prime_kv_cache_with(fixture, &mut context, "Hello\0world") + prime_memory_with(fixture, &mut context, "Hello\0world") .is_err_and(|error| error.downcast_ref::().is_some()), "an interior null byte must surface a tokenization error" ); assert_eq!( - prime_kv_cache_with(fixture, &mut context, &"word ".repeat(300)) + prime_memory_with(fixture, &mut context, &"word ".repeat(300)) .err() .and_then(|error| error.downcast_ref::().cloned()), Some(BatchAddError::BatchFull { n_batch: 256 }), @@ -30,10 +30,10 @@ fn prime_kv_cache_surfaces_each_underlying_error(fixture: &LlamaFixture<'_>) -> ); let filler = "word ".repeat(40); - let mut decode_result = prime_kv_cache_with(fixture, &mut context, &filler); + let mut decode_result = prime_memory_with(fixture, &mut context, &filler); let mut attempts = 0; while decode_result.is_ok() && attempts < 16 { - decode_result = prime_kv_cache_with(fixture, &mut context, &filler); + decode_result = prime_memory_with(fixture, &mut context, &filler); attempts += 1; } assert!( diff --git a/llama-cpp-bindings-tests/tests/kv_cache_sequence_edits.rs b/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs similarity index 78% rename from llama-cpp-bindings-tests/tests/kv_cache_sequence_edits.rs rename to llama-cpp-bindings-tests/tests/memory_sequence_edits.rs index 281a4a27e..e7dc44fd0 100644 --- a/llama-cpp-bindings-tests/tests/kv_cache_sequence_edits.rs +++ b/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs @@ -1,6 +1,6 @@ use anyhow::Result; -use llama_cpp_bindings::ClearKvCacheSeqError; -use llama_cpp_bindings_tests::prime_kv_cache::prime_kv_cache; +use llama_cpp_bindings::MemorySeqRmError; +use llama_cpp_bindings_tests::prime_memory::prime_memory; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -36,15 +36,12 @@ use llama_cpp_test_harness::llama_test; n_batch = 512, n_ubatch = 128, )] -fn clear_kv_cache_seq_accepts_an_explicit_range(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_rm_accepts_an_explicit_range(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; - prime_kv_cache(fixture, &mut context)?; + prime_memory(fixture, &mut context)?; - assert_eq!( - context.clear_kv_cache_seq(Some(0), Some(0), Some(1)), - Ok(()) - ); + assert_eq!(context.memory_seq_rm(Some(0), Some(0), Some(1)), Ok(())); Ok(()) } @@ -81,12 +78,12 @@ fn clear_kv_cache_seq_accepts_an_explicit_range(fixture: &LlamaFixture<'_>) -> R n_batch = 512, n_ubatch = 128, )] -fn copy_kv_cache_seq_succeeds(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_cp_succeeds(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; - prime_kv_cache(fixture, &mut context)?; + prime_memory(fixture, &mut context)?; - assert_eq!(context.copy_kv_cache_seq(0, 1, None, None), Ok(())); + assert_eq!(context.memory_seq_cp(0, 1, None, None), Ok(())); Ok(()) } @@ -123,12 +120,12 @@ fn copy_kv_cache_seq_succeeds(fixture: &LlamaFixture<'_>) -> Result<()> { n_batch = 512, n_ubatch = 128, )] -fn copy_kv_cache_seq_accepts_an_explicit_range(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_cp_accepts_an_explicit_range(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; - prime_kv_cache(fixture, &mut context)?; + prime_memory(fixture, &mut context)?; - assert_eq!(context.copy_kv_cache_seq(0, 2, Some(0), Some(1)), Ok(())); + assert_eq!(context.memory_seq_cp(0, 2, Some(0), Some(1)), Ok(())); Ok(()) } @@ -146,11 +143,11 @@ fn qwen35_recurrent_memory_refuses_to_remove_part_of_a_sequence( ) -> Result<()> { let mut context = fixture.build_context()?; - prime_kv_cache(fixture, &mut context)?; + prime_memory(fixture, &mut context)?; assert_eq!( - context.clear_kv_cache_seq(Some(0), Some(1), Some(2)), - Err(ClearKvCacheSeqError::PartialSequenceNotRemoved { + context.memory_seq_rm(Some(0), Some(1), Some(2)), + Err(MemorySeqRmError::PartialSequenceNotRemoved { seq_id: 0, p0: 1, p1: 2, diff --git a/llama-cpp-bindings-tests/tests/kv_cache_sequence_id_bounds.rs b/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs similarity index 80% rename from llama-cpp-bindings-tests/tests/kv_cache_sequence_id_bounds.rs rename to llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs index 879cba9d8..8350e08d1 100644 --- a/llama-cpp-bindings-tests/tests/kv_cache_sequence_id_bounds.rs +++ b/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use llama_cpp_bindings::error::ClearKvCacheSeqError; +use llama_cpp_bindings::error::MemorySeqRmError; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -35,14 +35,14 @@ use llama_cpp_test_harness::llama_test; n_batch = 512, n_ubatch = 128, )] -fn kv_cache_seq_pos_max_rejects_out_of_range_sequence(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_pos_max_rejects_out_of_range_sequence(fixture: &LlamaFixture<'_>) -> Result<()> { let context = fixture.build_context()?; - let result = context.kv_cache_seq_pos_max(999); + let result = context.memory_seq_pos_max(999); assert_eq!( result, - Err(llama_cpp_bindings::KvCacheSeqPosMaxError::SequenceIdOutOfRange { seq_id: 999 }) + Err(llama_cpp_bindings::MemorySeqPosMaxError::SequenceIdOutOfRange { seq_id: 999 }) ); Ok(()) @@ -80,16 +80,14 @@ fn kv_cache_seq_pos_max_rejects_out_of_range_sequence(fixture: &LlamaFixture<'_> n_batch = 512, n_ubatch = 128, )] -fn clear_kv_cache_seq_rejects_sequence_id_exceeding_i32_max( - fixture: &LlamaFixture<'_>, -) -> Result<()> { +fn memory_seq_rm_rejects_sequence_id_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; - let result = context.clear_kv_cache_seq(Some(u32::MAX), None, None); + let result = context.memory_seq_rm(Some(u32::MAX), None, None); assert_eq!( result.unwrap_err(), - ClearKvCacheSeqError::SeqIdTooLarge( + MemorySeqRmError::SeqIdTooLarge( i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") ) ); diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs index c593dc08b..d8af7d50e 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs @@ -20,7 +20,7 @@ use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; use llama_cpp_bindings_tests::media_question_chat_session::media_question_chat_session; -use llama_cpp_bindings_tests::prime_kv_cache_with::prime_kv_cache_with; +use llama_cpp_bindings_tests::prime_memory_with::prime_memory_with; use llama_cpp_bindings_tests::prompt_question::PROMPT_QUESTION; use llama_cpp_fault_injection::ArmedFault; use llama_cpp_fault_injection::injected_fault_message; @@ -193,7 +193,7 @@ fn eval_chunks_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_> chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, 0, true), Err(MtmdEvalError::NonPositiveBatchSize { requested: 0 }) ); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); + assert_eq!(llama_ctx.memory_seq_pos_max(0)?, -1); Ok(()) } @@ -273,7 +273,7 @@ fn eval_single_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_> first_chunk.eval_single(mtmd_ctx, &mut llama_ctx, 0, 0, 0, true), Err(MtmdEvalError::NonPositiveBatchSize { requested: 0 }) ); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); + assert_eq!(llama_ctx.memory_seq_pos_max(0)?, -1); Ok(()) } @@ -311,7 +311,7 @@ fn eval_single_rejects_a_batch_larger_than_the_context_batch_before_evaluating( context_max, }) ); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); + assert_eq!(llama_ctx.memory_seq_pos_max(0)?, -1); Ok(()) } @@ -344,7 +344,7 @@ fn eval_chunks_rejects_a_sequence_the_context_does_not_have_instead_of_aborting( n_seq_max, })) ); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); + assert_eq!(llama_ctx.memory_seq_pos_max(0)?, -1); Ok(()) } @@ -402,7 +402,7 @@ fn eval_single_rejects_a_micro_batch_smaller_than_the_fitted_one_before_evaluati fitted_micro_batch_tokens, }) ); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); + assert_eq!(llama_ctx.memory_seq_pos_max(0)?, -1); Ok(()) } @@ -454,7 +454,7 @@ fn classifier_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_>) )) )); assert_eq!(classifier.usage().prompt_tokens, 0); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); + assert_eq!(llama_ctx.memory_seq_pos_max(0)?, -1); Ok(()) } @@ -617,7 +617,7 @@ fn gemma3_eval_chunks_rejects_a_non_causal_image_larger_than_the_micro_batch_bef } )) ); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); + assert_eq!(llama_ctx.memory_seq_pos_max(0)?, -1); Ok(()) } @@ -707,7 +707,7 @@ fn gemma3_classifier_leaves_the_prompt_untouched_when_a_non_causal_image_exceeds )) if micro_batch_tokens == llama_ctx.n_ubatch() )); assert_eq!(classifier.usage().prompt_tokens, 0); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); + assert_eq!(llama_ctx.memory_seq_pos_max(0)?, -1); Ok(()) } @@ -735,9 +735,9 @@ fn classifier_reports_an_image_the_context_has_no_room_left_for( (*fixture.context_params).into_llama_context_params(), )?; - prime_kv_cache_with(fixture, &mut llama_ctx, &"hello ".repeat(230))?; + prime_memory_with(fixture, &mut llama_ctx, &"hello ".repeat(230))?; - let primed_positions = llama_ctx.kv_cache_seq_pos_max(0)? + 1; + let primed_positions = llama_ctx.memory_seq_pos_max(0)? + 1; let mut classifier = SampledTokenClassifier::undeterminable(fixture.model); assert!( @@ -783,9 +783,9 @@ fn eval_chunks_reports_an_image_the_context_has_no_room_left_for( let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; let mut llama_ctx = fixture.build_context()?; - prime_kv_cache_with(fixture, &mut llama_ctx, &"hello ".repeat(230))?; + prime_memory_with(fixture, &mut llama_ctx, &"hello ".repeat(230))?; - let primed_positions = llama_ctx.kv_cache_seq_pos_max(0)? + 1; + let primed_positions = llama_ctx.memory_seq_pos_max(0)? + 1; assert_eq!( chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, primed_positions, 0, 256, true), diff --git a/llama-cpp-bindings-tests/tests/reranking.rs b/llama-cpp-bindings-tests/tests/reranking.rs index 0eea7acbb..53424dba6 100644 --- a/llama-cpp-bindings-tests/tests/reranking.rs +++ b/llama-cpp-bindings-tests/tests/reranking.rs @@ -88,7 +88,7 @@ fn reranking_produces_scores(fixture: &LlamaFixture<'_>) -> Result<()> { assert_eq!(classifier.pending_prompt_tokens(), total_token_count); assert_eq!(classifier.usage().prompt_tokens, 0); - ctx.clear_kv_cache(); + ctx.memory_clear(); ctx.decode(&batch) .with_context(|| "llama_decode() failed")?; diff --git a/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs b/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs index 2300675a1..20d4c2c0f 100644 --- a/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs +++ b/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs @@ -31,32 +31,6 @@ fn accept_many_iterates_over_borrowed_tokens(fixture: &LlamaFixture<'_>) -> Resu Ok(()) } -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn with_tokens_returns_self_after_accepting_each_token(fixture: &LlamaFixture<'_>) -> Result<()> { - let sampler = LlamaSampler::chain([LlamaSampler::greedy()?])?; - let tokens = [ - fixture - .model - .token_bos() - .context("model declares a BOS token")?, - fixture - .model - .token_eos() - .context("model declares an EOS token")?, - ]; - - let _consumed = sampler.with_tokens(tokens.iter().copied())?; - - Ok(()) -} - #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, diff --git a/llama-cpp-bindings-tests/tests/state_snapshots.rs b/llama-cpp-bindings-tests/tests/state_snapshots.rs index 9c04e2988..e538bfb47 100644 --- a/llama-cpp-bindings-tests/tests/state_snapshots.rs +++ b/llama-cpp-bindings-tests/tests/state_snapshots.rs @@ -38,10 +38,10 @@ use llama_cpp_test_harness::llama_test; n_batch = 512, n_ubatch = 128, )] -fn get_state_size_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { +fn state_get_size_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { let context = fixture.build_context()?; - assert!(context.get_state_size() > 0); + assert!(context.state_get_size() > 0); Ok(()) } @@ -78,7 +78,7 @@ fn get_state_size_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { n_batch = 512, n_ubatch = 128, )] -fn set_state_data_rejects_a_truncated_snapshot(fixture: &LlamaFixture<'_>) -> Result<()> { +fn state_set_data_rejects_a_truncated_snapshot(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; let tokens = @@ -89,12 +89,12 @@ fn set_state_data_rejects_a_truncated_snapshot(fixture: &LlamaFixture<'_>) -> Re batch.add_sequence(&tokens, 0, false)?; context.decode(&batch)?; - let mut state_data = vec![0u8; context.get_state_size()]; - let bytes_copied = unsafe { context.copy_state_data(&mut state_data) }?; + let mut state_data = vec![0u8; context.state_get_size()]; + let bytes_copied = unsafe { context.state_get_data(&mut state_data) }?; state_data.truncate(bytes_copied / 2); assert_eq!( - unsafe { context.set_state_data(&state_data) }, + unsafe { context.state_set_data(&state_data) }, Err( llama_cpp_bindings::context::state_data_error::StateDataError::NothingRestored { provided_bytes: state_data.len(), @@ -114,7 +114,7 @@ fn set_state_data_rejects_a_truncated_snapshot(fixture: &LlamaFixture<'_>) -> Re n_batch = 512, n_ubatch = 128, )] -fn copy_state_data_and_set_state_data_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> { +fn state_get_data_and_state_set_data_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; let tokens = @@ -125,12 +125,12 @@ fn copy_state_data_and_set_state_data_roundtrip(fixture: &LlamaFixture<'_>) -> R batch.add_sequence(&tokens, 0, false)?; context.decode(&batch)?; - let state_size = context.get_state_size(); + let state_size = context.state_get_size(); let mut state_data = vec![0u8; state_size]; - let bytes_copied = unsafe { context.copy_state_data(&mut state_data) }?; + let bytes_copied = unsafe { context.state_get_data(&mut state_data) }?; assert!(bytes_copied > 0); - let bytes_read = unsafe { context.set_state_data(&state_data) }?; + let bytes_read = unsafe { context.state_set_data(&state_data) }?; assert_eq!( bytes_read, bytes_copied, "restoring the state must consume exactly the bytes the snapshot produced" diff --git a/llama-cpp-bindings/src/backend_device_index.rs b/llama-cpp-bindings/src/backend_device_index.rs new file mode 100644 index 000000000..b88083e3d --- /dev/null +++ b/llama-cpp-bindings/src/backend_device_index.rs @@ -0,0 +1,12 @@ +/// The index of `device` among the backend devices ggml has registered. +#[must_use] +pub fn backend_device_index(device: llama_cpp_bindings_sys::ggml_backend_dev_t) -> Option { + let device_count = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_count() }; + + (0..device_count).position(|device_index| { + let registered_device = + unsafe { llama_cpp_bindings_sys::ggml_backend_dev_get(device_index) }; + + registered_device == device + }) +} diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index 33167fdda..58d5c61d5 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -365,7 +365,6 @@ unsafe fn logits_slice<'logits>( pub mod context_limits; pub mod decoded_output_rows; pub mod decoded_outputs; -pub mod kv_cache; pub mod kv_cache_type; pub mod llama_attention_type; pub mod llama_pooling_type; @@ -373,6 +372,7 @@ pub mod llama_state_seq_flags; pub mod load_seq_state_error; pub mod load_session_error; pub mod loaded_seq_state; +pub mod memory; pub mod params; pub mod rope_scaling_type; pub mod save_seq_state_error; @@ -497,8 +497,7 @@ impl<'model> LlamaContext<'model> { } fn decodes_batches_in_one_micro_batch(&self) -> bool { - let keeps_no_memory = - unsafe { llama_cpp_bindings_sys::llama_get_memory(self.context.as_ptr()) }.is_null(); + let keeps_no_memory = !self.has_memory(); let attends_causally = unsafe { llama_cpp_bindings_sys::llama_get_causal_attn(self.context.as_ptr()) }; diff --git a/llama-cpp-bindings/src/context/kv_cache.rs b/llama-cpp-bindings/src/context/memory.rs similarity index 72% rename from llama-cpp-bindings/src/context/kv_cache.rs rename to llama-cpp-bindings/src/context/memory.rs index f3cac5a5a..a3c8af08a 100644 --- a/llama-cpp-bindings/src/context/kv_cache.rs +++ b/llama-cpp-bindings/src/context/memory.rs @@ -5,28 +5,27 @@ use std::ptr; use crate::context::LlamaContext; use crate::error::{ - ClearKvCacheSeqError, CopyKvCacheSeqError, KvCacheSeqAddError, KvCacheSeqDivError, - KvCacheSeqPosMaxError, + MemorySeqAddError, MemorySeqCpError, MemorySeqDivError, MemorySeqPosMaxError, MemorySeqRmError, }; use llama_cpp_ffi_status::read_and_free_cpp_string; -fn kv_cache_seq_add_status_to_result( +fn memory_seq_add_status_to_result( status: llama_cpp_bindings_sys::llama_rs_memory_seq_add_status, out_error: *mut c_char, -) -> Result<(), KvCacheSeqAddError> { +) -> Result<(), MemorySeqAddError> { match status { llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_OK => Ok(()), llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_INCOMPATIBLE_ROPE_TYPE => { - Err(KvCacheSeqAddError::IncompatibleRopeType) + Err(MemorySeqAddError::IncompatibleRopeType) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_MEM => { - Err(KvCacheSeqAddError::MemoryHandleUnavailable) + Err(MemorySeqAddError::MemoryHandleUnavailable) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_ERROR_STRING_ALLOCATION_FAILED => { - Err(KvCacheSeqAddError::NotEnoughMemory) + Err(MemorySeqAddError::NotEnoughMemory) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_LLAMA_CPP_OUT_OF_MEMORY => { - Err(KvCacheSeqAddError::LlamaCppOutOfMemory) + Err(MemorySeqAddError::LlamaCppOutOfMemory) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION => { let message = unsafe { @@ -36,7 +35,7 @@ fn kv_cache_seq_add_status_to_result( "reported a thrown C++ exception without an error message", ) }?; - Err(KvCacheSeqAddError::Reported { message }) + Err(MemorySeqAddError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_CTX_ARG => { Err(crate::FfiContractError { @@ -60,23 +59,23 @@ fn kv_cache_seq_add_status_to_result( } } -fn kv_cache_seq_div_status_to_result( +fn memory_seq_div_status_to_result( status: llama_cpp_bindings_sys::llama_rs_memory_seq_div_status, out_error: *mut c_char, -) -> Result<(), KvCacheSeqDivError> { +) -> Result<(), MemorySeqDivError> { match status { llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_OK => Ok(()), llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_INCOMPATIBLE_ROPE_TYPE => { - Err(KvCacheSeqDivError::IncompatibleRopeType) + Err(MemorySeqDivError::IncompatibleRopeType) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_MEM => { - Err(KvCacheSeqDivError::MemoryHandleUnavailable) + Err(MemorySeqDivError::MemoryHandleUnavailable) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_ERROR_STRING_ALLOCATION_FAILED => { - Err(KvCacheSeqDivError::NotEnoughMemory) + Err(MemorySeqDivError::NotEnoughMemory) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_LLAMA_CPP_OUT_OF_MEMORY => { - Err(KvCacheSeqDivError::LlamaCppOutOfMemory) + Err(MemorySeqDivError::LlamaCppOutOfMemory) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_LLAMA_CPP_THREW_CXX_EXCEPTION => { let message = unsafe { @@ -86,7 +85,7 @@ fn kv_cache_seq_div_status_to_result( "reported a thrown C++ exception without an error message", ) }?; - Err(KvCacheSeqDivError::Reported { message }) + Err(MemorySeqDivError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_CTX_ARG => { Err(crate::FfiContractError { @@ -110,12 +109,12 @@ fn kv_cache_seq_div_status_to_result( } } -fn kv_cache_seq_pos_max_status_to_result( +fn memory_seq_pos_max_status_to_result( status: llama_cpp_bindings_sys::llama_rs_memory_seq_pos_max_status, position: i32, seq_id: i32, out_error: *mut c_char, -) -> Result { +) -> Result { match status { llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_OK => Ok(position), llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_CTX_ARG => { @@ -140,16 +139,16 @@ fn kv_cache_seq_pos_max_status_to_result( .into()) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_MEM => { - Err(KvCacheSeqPosMaxError::MemoryHandleUnavailable) + Err(MemorySeqPosMaxError::MemoryHandleUnavailable) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_SEQ_ID_OUT_OF_RANGE => { - Err(KvCacheSeqPosMaxError::SequenceIdOutOfRange { seq_id }) + Err(MemorySeqPosMaxError::SequenceIdOutOfRange { seq_id }) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_ERROR_STRING_ALLOCATION_FAILED => { - Err(KvCacheSeqPosMaxError::NotEnoughMemory) + Err(MemorySeqPosMaxError::NotEnoughMemory) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_LLAMA_CPP_OUT_OF_MEMORY => { - Err(KvCacheSeqPosMaxError::LlamaCppOutOfMemory) + Err(MemorySeqPosMaxError::LlamaCppOutOfMemory) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_LLAMA_CPP_THREW_CXX_EXCEPTION => { let message = unsafe { @@ -159,7 +158,7 @@ fn kv_cache_seq_pos_max_status_to_result( "reported a thrown C++ exception without an error message", ) }?; - Err(KvCacheSeqPosMaxError::Reported { message }) + Err(MemorySeqPosMaxError::Reported { message }) } other => Err(crate::FfiStatusError { operation: "llama_rs_memory_seq_pos_max", @@ -183,26 +182,26 @@ impl LlamaContext<'_> { fn required_memory_handle( &self, - ) -> Result { + ) -> Result { self.memory_handle() - .ok_or(CopyKvCacheSeqError::MemoryHandleUnavailable) + .ok_or(MemorySeqCpError::MemoryHandleUnavailable) } /// # Errors /// If either position exceeds [`i32::MAX`], or the context has no memory module. - pub fn copy_kv_cache_seq( + pub fn memory_seq_cp( &mut self, src: i32, dest: i32, p0: Option, p1: Option, - ) -> Result<(), CopyKvCacheSeqError> { + ) -> Result<(), MemorySeqCpError> { let p0 = p0 .map_or(Ok(-1), i32::try_from) - .map_err(CopyKvCacheSeqError::P0TooLarge)?; + .map_err(MemorySeqCpError::P0TooLarge)?; let p1 = p1 .map_or(Ok(-1), i32::try_from) - .map_err(CopyKvCacheSeqError::P1TooLarge)?; + .map_err(MemorySeqCpError::P1TooLarge)?; let mem = self.required_memory_handle()?; unsafe { llama_cpp_bindings_sys::llama_memory_seq_cp(mem, src, dest, p0, p1) }; @@ -213,21 +212,21 @@ impl LlamaContext<'_> { /// If the sequence id or either position exceeds [`i32::MAX`], or llama.cpp reports /// that the partial sequence could not be removed. A context without a memory module /// holds no KV cache, so the removal trivially succeeds. - pub fn clear_kv_cache_seq( + pub fn memory_seq_rm( &mut self, seq_id: Option, p0: Option, p1: Option, - ) -> Result<(), ClearKvCacheSeqError> { + ) -> Result<(), MemorySeqRmError> { let seq_id = seq_id .map_or(Ok(-1), i32::try_from) - .map_err(ClearKvCacheSeqError::SeqIdTooLarge)?; + .map_err(MemorySeqRmError::SeqIdTooLarge)?; let p0 = p0 .map_or(Ok(-1), i32::try_from) - .map_err(ClearKvCacheSeqError::P0TooLarge)?; + .map_err(MemorySeqRmError::P0TooLarge)?; let p1 = p1 .map_or(Ok(-1), i32::try_from) - .map_err(ClearKvCacheSeqError::P1TooLarge)?; + .map_err(MemorySeqRmError::P1TooLarge)?; let Some(mem) = self.memory_handle() else { return Ok(()); }; @@ -236,10 +235,10 @@ impl LlamaContext<'_> { return Ok(()); } - Err(ClearKvCacheSeqError::PartialSequenceNotRemoved { seq_id, p0, p1 }) + Err(MemorySeqRmError::PartialSequenceNotRemoved { seq_id, p0, p1 }) } - pub fn clear_kv_cache(&mut self) { + pub fn memory_clear(&mut self) { let Some(mem) = self.memory_handle() else { return; }; @@ -247,7 +246,7 @@ impl LlamaContext<'_> { unsafe { llama_cpp_bindings_sys::llama_memory_clear(mem, clear_data_buffers) }; } - pub fn kv_cache_seq_keep(&mut self, seq_id: i32) { + pub fn memory_seq_keep(&mut self, seq_id: i32) { let Some(mem) = self.memory_handle() else { return; }; @@ -256,19 +255,19 @@ impl LlamaContext<'_> { /// # Errors /// If either position exceeds [`i32::MAX`], or the underlying memory operation reports a failure. - pub fn kv_cache_seq_add( + pub fn memory_seq_add( &mut self, seq_id: i32, p0: Option, p1: Option, delta: i32, - ) -> Result<(), KvCacheSeqAddError> { + ) -> Result<(), MemorySeqAddError> { let p0 = p0 .map_or(Ok(-1), i32::try_from) - .map_err(KvCacheSeqAddError::P0TooLarge)?; + .map_err(MemorySeqAddError::P0TooLarge)?; let p1 = p1 .map_or(Ok(-1), i32::try_from) - .map_err(KvCacheSeqAddError::P1TooLarge)?; + .map_err(MemorySeqAddError::P1TooLarge)?; let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_memory_seq_add( @@ -280,24 +279,24 @@ impl LlamaContext<'_> { &raw mut out_error, ) }; - kv_cache_seq_add_status_to_result(status, out_error) + memory_seq_add_status_to_result(status, out_error) } /// # Errors /// If either position exceeds [`i32::MAX`], or the underlying memory operation reports a failure. - pub fn kv_cache_seq_div( + pub fn memory_seq_div( &mut self, seq_id: i32, p0: Option, p1: Option, divisor: NonZeroU8, - ) -> Result<(), KvCacheSeqDivError> { + ) -> Result<(), MemorySeqDivError> { let p0 = p0 .map_or(Ok(-1), i32::try_from) - .map_err(KvCacheSeqDivError::P0TooLarge)?; + .map_err(MemorySeqDivError::P0TooLarge)?; let p1 = p1 .map_or(Ok(-1), i32::try_from) - .map_err(KvCacheSeqDivError::P1TooLarge)?; + .map_err(MemorySeqDivError::P1TooLarge)?; let divisor = c_int::from(divisor.get()); let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { @@ -310,13 +309,13 @@ impl LlamaContext<'_> { &raw mut out_error, ) }; - kv_cache_seq_div_status_to_result(status, out_error) + memory_seq_div_status_to_result(status, out_error) } /// # Errors /// - /// Returns [`KvCacheSeqPosMaxError`] if the sequence does not exist or the memory lookup fails. - pub fn kv_cache_seq_pos_max(&self, seq_id: i32) -> Result { + /// Returns [`MemorySeqPosMaxError`] if the sequence does not exist or the memory lookup fails. + pub fn memory_seq_pos_max(&self, seq_id: i32) -> Result { let mut position = -1; let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { @@ -328,20 +327,20 @@ impl LlamaContext<'_> { ) }; - kv_cache_seq_pos_max_status_to_result(status, position, seq_id, out_error) + memory_seq_pos_max_status_to_result(status, position, seq_id, out_error) } } #[cfg(test)] -mod kv_cache_seq_add_status_to_result_tests { +mod memory_seq_add_status_to_result_tests { use std::ptr; - use super::kv_cache_seq_add_status_to_result; - use crate::error::KvCacheSeqAddError; + use super::memory_seq_add_status_to_result; + use crate::error::MemorySeqAddError; #[test] fn add_ok_status_maps_to_ok() { - let result = kv_cache_seq_add_status_to_result( + let result = memory_seq_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_OK, ptr::null_mut(), ); @@ -352,40 +351,40 @@ mod kv_cache_seq_add_status_to_result_tests { #[test] fn add_incompatible_rope_type_status_maps_to_incompatible_rope_type() { assert_eq!( - kv_cache_seq_add_status_to_result( + memory_seq_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_INCOMPATIBLE_ROPE_TYPE, ptr::null_mut(), ), - Err(KvCacheSeqAddError::IncompatibleRopeType) + Err(MemorySeqAddError::IncompatibleRopeType) ); } #[test] fn add_null_mem_status_maps_to_memory_handle_unavailable() { assert_eq!( - kv_cache_seq_add_status_to_result( + memory_seq_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_MEM, ptr::null_mut(), ), - Err(KvCacheSeqAddError::MemoryHandleUnavailable) + Err(MemorySeqAddError::MemoryHandleUnavailable) ); } #[test] fn add_allocation_failed_status_maps_to_not_enough_memory() { assert_eq!( - kv_cache_seq_add_status_to_result( + memory_seq_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_ERROR_STRING_ALLOCATION_FAILED, ptr::null_mut(), ), - Err(KvCacheSeqAddError::NotEnoughMemory) + Err(MemorySeqAddError::NotEnoughMemory) ); } #[test] fn add_llama_cpp_exception_status_without_a_message_is_a_contract_error_with_unknown_message() { assert_eq!( - kv_cache_seq_add_status_to_result( + memory_seq_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), ), @@ -403,11 +402,11 @@ mod kv_cache_seq_add_status_to_result_tests { unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"shift threw".as_ptr()) }; assert_eq!( - kv_cache_seq_add_status_to_result( + memory_seq_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION, out_error, ), - Err(KvCacheSeqAddError::Reported { + Err(MemorySeqAddError::Reported { message: "shift threw".to_owned() }) ); @@ -415,11 +414,11 @@ mod kv_cache_seq_add_status_to_result_tests { #[test] fn add_unknown_status_is_preserved() { - let result = kv_cache_seq_add_status_to_result(255, ptr::null_mut()); + let result = memory_seq_add_status_to_result(255, ptr::null_mut()); assert_eq!( result, - Err(KvCacheSeqAddError::FfiStatus(crate::FfiStatusError { + Err(MemorySeqAddError::FfiStatus(crate::FfiStatusError { operation: "llama_rs_memory_seq_add", code: 255, })) @@ -427,8 +426,8 @@ mod kv_cache_seq_add_status_to_result_tests { } #[test] - fn kv_cache_seq_add_status_to_result_maps_null_ctx_arg() { - let outcome = kv_cache_seq_add_status_to_result( + fn memory_seq_add_status_to_result_maps_null_ctx_arg() { + let outcome = memory_seq_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_CTX_ARG, ptr::null_mut(), ); @@ -445,8 +444,8 @@ mod kv_cache_seq_add_status_to_result_tests { } #[test] - fn kv_cache_seq_add_status_to_result_maps_null_model() { - let outcome = kv_cache_seq_add_status_to_result( + fn memory_seq_add_status_to_result_maps_null_model() { + let outcome = memory_seq_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_MODEL, ptr::null_mut(), ); @@ -463,25 +462,25 @@ mod kv_cache_seq_add_status_to_result_tests { } #[test] - fn kv_cache_seq_add_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = kv_cache_seq_add_status_to_result( + fn memory_seq_add_status_to_result_maps_llama_cpp_out_of_memory() { + let outcome = memory_seq_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), ); - assert_eq!(outcome.err(), Some(KvCacheSeqAddError::LlamaCppOutOfMemory)); + assert_eq!(outcome.err(), Some(MemorySeqAddError::LlamaCppOutOfMemory)); } } #[cfg(test)] -mod kv_cache_seq_div_status_to_result_tests { +mod memory_seq_div_status_to_result_tests { use std::ptr; - use super::kv_cache_seq_div_status_to_result; - use crate::error::KvCacheSeqDivError; + use super::memory_seq_div_status_to_result; + use crate::error::MemorySeqDivError; #[test] fn div_ok_status_maps_to_ok() { - let result = kv_cache_seq_div_status_to_result( + let result = memory_seq_div_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_OK, ptr::null_mut(), ); @@ -492,40 +491,40 @@ mod kv_cache_seq_div_status_to_result_tests { #[test] fn div_incompatible_rope_type_status_maps_to_incompatible_rope_type() { assert_eq!( - kv_cache_seq_div_status_to_result( + memory_seq_div_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_INCOMPATIBLE_ROPE_TYPE, ptr::null_mut(), ), - Err(KvCacheSeqDivError::IncompatibleRopeType) + Err(MemorySeqDivError::IncompatibleRopeType) ); } #[test] fn div_null_mem_status_maps_to_memory_handle_unavailable() { assert_eq!( - kv_cache_seq_div_status_to_result( + memory_seq_div_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_MEM, ptr::null_mut(), ), - Err(KvCacheSeqDivError::MemoryHandleUnavailable) + Err(MemorySeqDivError::MemoryHandleUnavailable) ); } #[test] fn div_allocation_failed_status_maps_to_not_enough_memory() { assert_eq!( - kv_cache_seq_div_status_to_result( + memory_seq_div_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_ERROR_STRING_ALLOCATION_FAILED, ptr::null_mut(), ), - Err(KvCacheSeqDivError::NotEnoughMemory) + Err(MemorySeqDivError::NotEnoughMemory) ); } #[test] fn div_llama_cpp_exception_status_without_a_message_is_a_contract_error_with_unknown_message() { assert_eq!( - kv_cache_seq_div_status_to_result( + memory_seq_div_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), ), @@ -543,11 +542,11 @@ mod kv_cache_seq_div_status_to_result_tests { unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"division threw".as_ptr()) }; assert_eq!( - kv_cache_seq_div_status_to_result( + memory_seq_div_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_LLAMA_CPP_THREW_CXX_EXCEPTION, out_error, ), - Err(KvCacheSeqDivError::Reported { + Err(MemorySeqDivError::Reported { message: "division threw".to_owned() }) ); @@ -555,11 +554,11 @@ mod kv_cache_seq_div_status_to_result_tests { #[test] fn div_unknown_status_is_preserved() { - let result = kv_cache_seq_div_status_to_result(255, ptr::null_mut()); + let result = memory_seq_div_status_to_result(255, ptr::null_mut()); assert_eq!( result, - Err(KvCacheSeqDivError::FfiStatus(crate::FfiStatusError { + Err(MemorySeqDivError::FfiStatus(crate::FfiStatusError { operation: "llama_rs_memory_seq_div", code: 255, })) @@ -567,8 +566,8 @@ mod kv_cache_seq_div_status_to_result_tests { } #[test] - fn kv_cache_seq_div_status_to_result_maps_null_ctx_arg() { - let outcome = kv_cache_seq_div_status_to_result( + fn memory_seq_div_status_to_result_maps_null_ctx_arg() { + let outcome = memory_seq_div_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_CTX_ARG, ptr::null_mut(), ); @@ -585,8 +584,8 @@ mod kv_cache_seq_div_status_to_result_tests { } #[test] - fn kv_cache_seq_div_status_to_result_maps_null_model() { - let outcome = kv_cache_seq_div_status_to_result( + fn memory_seq_div_status_to_result_maps_null_model() { + let outcome = memory_seq_div_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_MODEL, ptr::null_mut(), ); @@ -603,26 +602,26 @@ mod kv_cache_seq_div_status_to_result_tests { } #[test] - fn kv_cache_seq_div_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = kv_cache_seq_div_status_to_result( + fn memory_seq_div_status_to_result_maps_llama_cpp_out_of_memory() { + let outcome = memory_seq_div_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), ); - assert_eq!(outcome.err(), Some(KvCacheSeqDivError::LlamaCppOutOfMemory)); + assert_eq!(outcome.err(), Some(MemorySeqDivError::LlamaCppOutOfMemory)); } } #[cfg(test)] -mod kv_cache_seq_pos_max_status_to_result_tests { +mod memory_seq_pos_max_status_to_result_tests { use std::ptr; - use super::kv_cache_seq_pos_max_status_to_result; - use crate::error::KvCacheSeqPosMaxError; + use super::memory_seq_pos_max_status_to_result; + use crate::error::MemorySeqPosMaxError; #[test] fn seq_pos_max_ok_status_returns_position() { assert_eq!( - kv_cache_seq_pos_max_status_to_result( + memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_OK, 17, 2, @@ -635,100 +634,94 @@ mod kv_cache_seq_pos_max_status_to_result_tests { #[test] fn seq_pos_max_null_context_status_is_contract_error() { assert_eq!( - kv_cache_seq_pos_max_status_to_result( + memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_CTX_ARG, -1, 2, ptr::null_mut(), ), - Err(KvCacheSeqPosMaxError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_memory_seq_pos_max", - detail: "context pointer was null", - } - )) + Err(MemorySeqPosMaxError::FfiContract(crate::FfiContractError { + operation: "llama_rs_memory_seq_pos_max", + detail: "context pointer was null", + })) ); } #[test] fn seq_pos_max_null_output_position_status_is_contract_error() { assert_eq!( - kv_cache_seq_pos_max_status_to_result( + memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_OUT_POSITION_ARG, -1, 2, ptr::null_mut(), ), - Err(KvCacheSeqPosMaxError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_memory_seq_pos_max", - detail: "output position pointer was null", - } - )) + Err(MemorySeqPosMaxError::FfiContract(crate::FfiContractError { + operation: "llama_rs_memory_seq_pos_max", + detail: "output position pointer was null", + })) ); } #[test] fn seq_pos_max_null_output_error_status_is_contract_error() { assert_eq!( - kv_cache_seq_pos_max_status_to_result( + memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_OUT_ERROR_ARG, -1, 2, ptr::null_mut(), ), - Err(KvCacheSeqPosMaxError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_memory_seq_pos_max", - detail: "output error pointer was null", - } - )) + Err(MemorySeqPosMaxError::FfiContract(crate::FfiContractError { + operation: "llama_rs_memory_seq_pos_max", + detail: "output error pointer was null", + })) ); } #[test] fn seq_pos_max_null_memory_status_maps_to_memory_handle_unavailable() { assert_eq!( - kv_cache_seq_pos_max_status_to_result( + memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_MEM, -1, 2, ptr::null_mut(), ), - Err(KvCacheSeqPosMaxError::MemoryHandleUnavailable) + Err(MemorySeqPosMaxError::MemoryHandleUnavailable) ); } #[test] fn seq_pos_max_out_of_range_status_preserves_sequence_id() { assert_eq!( - kv_cache_seq_pos_max_status_to_result( + memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_SEQ_ID_OUT_OF_RANGE, -1, 27, ptr::null_mut(), ), - Err(KvCacheSeqPosMaxError::SequenceIdOutOfRange { seq_id: 27 }) + Err(MemorySeqPosMaxError::SequenceIdOutOfRange { seq_id: 27 }) ); } #[test] fn seq_pos_max_allocation_failed_status_maps_to_not_enough_memory() { assert_eq!( - kv_cache_seq_pos_max_status_to_result( + memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_ERROR_STRING_ALLOCATION_FAILED, -1, 2, ptr::null_mut(), ), - Err(KvCacheSeqPosMaxError::NotEnoughMemory) + Err(MemorySeqPosMaxError::NotEnoughMemory) ); } #[test] fn seq_pos_max_llama_cpp_exception_status_without_a_message_is_a_contract_error_error() { assert_eq!( - kv_cache_seq_pos_max_status_to_result( + memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_LLAMA_CPP_THREW_CXX_EXCEPTION, -1, 2, @@ -749,13 +742,13 @@ mod kv_cache_seq_pos_max_status_to_result_tests { }; assert_eq!( - kv_cache_seq_pos_max_status_to_result( + memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_LLAMA_CPP_THREW_CXX_EXCEPTION, -1, 2, out_error, ), - Err(KvCacheSeqPosMaxError::Reported { + Err(MemorySeqPosMaxError::Reported { message: "position lookup threw".to_owned() }) ); @@ -764,8 +757,8 @@ mod kv_cache_seq_pos_max_status_to_result_tests { #[test] fn seq_pos_max_unknown_status_is_preserved() { assert_eq!( - kv_cache_seq_pos_max_status_to_result(255, -1, 2, ptr::null_mut(),), - Err(KvCacheSeqPosMaxError::FfiStatus(crate::FfiStatusError { + memory_seq_pos_max_status_to_result(255, -1, 2, ptr::null_mut(),), + Err(MemorySeqPosMaxError::FfiStatus(crate::FfiStatusError { operation: "llama_rs_memory_seq_pos_max", code: 255, })) @@ -773,8 +766,8 @@ mod kv_cache_seq_pos_max_status_to_result_tests { } #[test] - fn kv_cache_seq_pos_max_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = kv_cache_seq_pos_max_status_to_result( + fn memory_seq_pos_max_status_to_result_maps_llama_cpp_out_of_memory() { + let outcome = memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_LLAMA_CPP_OUT_OF_MEMORY, 0, 0, @@ -782,7 +775,7 @@ mod kv_cache_seq_pos_max_status_to_result_tests { ); assert_eq!( outcome.err(), - Some(KvCacheSeqPosMaxError::LlamaCppOutOfMemory) + Some(MemorySeqPosMaxError::LlamaCppOutOfMemory) ); } } diff --git a/llama-cpp-bindings/src/context/session.rs b/llama-cpp-bindings/src/context/session.rs index 5cc36d8ae..25bd6a2b3 100644 --- a/llama-cpp-bindings/src/context/session.rs +++ b/llama-cpp-bindings/src/context/session.rs @@ -265,7 +265,7 @@ impl LlamaContext<'_> { } #[must_use] - pub fn get_state_size(&self) -> usize { + pub fn state_get_size(&self) -> usize { unsafe { llama_cpp_bindings_sys::llama_state_get_size(self.context.as_ptr()) } } @@ -277,7 +277,7 @@ impl LlamaContext<'_> { /// # Safety /// /// The `dest` buffer must be large enough to hold the complete state data. - pub unsafe fn copy_state_data(&self, dest: &mut [u8]) -> Result { + pub unsafe fn state_get_data(&self, dest: &mut [u8]) -> Result { let mut byte_count = 0usize; let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { @@ -295,14 +295,14 @@ impl LlamaContext<'_> { /// # Safety /// - /// The `src` buffer must contain data previously obtained from [`copy_state_data`](Self::copy_state_data) + /// The `src` buffer must contain data previously obtained from [`state_get_data`](Self::state_get_data) /// on a compatible context (same model and parameters). Passing arbitrary or corrupted bytes /// will lead to undefined behavior. /// /// # Errors /// /// Returns [`StateDataError`] when the llama.cpp deserializer rejects the buffer. - pub unsafe fn set_state_data(&mut self, src: &[u8]) -> Result { + pub unsafe fn state_set_data(&mut self, src: &[u8]) -> Result { let mut byte_count = 0usize; let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { diff --git a/llama-cpp-bindings/src/error.rs b/llama-cpp-bindings/src/error.rs index 2c95040a0..5f8caf431 100644 --- a/llama-cpp-bindings/src/error.rs +++ b/llama-cpp-bindings/src/error.rs @@ -3,8 +3,6 @@ pub mod bracketed_args_failure; pub mod chat_session_error; pub mod chat_template_error; pub mod chat_templates_error; -pub mod clear_kv_cache_seq_error; -pub mod copy_kv_cache_seq_error; pub mod decode_error; pub mod embeddings_error; pub mod encode_error; @@ -16,15 +14,17 @@ pub mod ingest_error; pub mod json_object_failure; pub mod json_schema_to_grammar_error; pub mod key_value_xml_tags_failure; -pub mod kv_cache_seq_add_error; -pub mod kv_cache_seq_div_error; -pub mod kv_cache_seq_pos_max_error; pub mod llama_context_load_error; pub mod llama_cpp_error; pub mod llama_lora_adapter_init_error; pub mod llama_lora_adapters_error; pub mod llama_model_load_error; pub mod logits_error; +pub mod memory_seq_add_error; +pub mod memory_seq_cp_error; +pub mod memory_seq_div_error; +pub mod memory_seq_pos_max_error; +pub mod memory_seq_rm_error; pub mod meta_val_error; pub mod model_params_error; pub mod new_llama_chat_message_error; @@ -39,7 +39,6 @@ pub mod sequence_id_out_of_range; pub mod string_to_token_error; pub mod token_attr_error; pub mod token_outside_vocabulary; -pub mod token_sampling_error; pub mod token_to_string_error; pub mod tool_call_format_failure; pub mod xml_function_tags_failure; @@ -51,8 +50,6 @@ pub use bracketed_args_failure::BracketedArgsFailure; pub use chat_session_error::ChatSessionError; pub use chat_template_error::ChatTemplateError; pub use chat_templates_error::ChatTemplatesError; -pub use clear_kv_cache_seq_error::ClearKvCacheSeqError; -pub use copy_kv_cache_seq_error::CopyKvCacheSeqError; pub use decode_error::DecodeError; pub use embeddings_error::EmbeddingsError; pub use encode_error::EncodeError; @@ -64,15 +61,17 @@ pub use ingest_error::IngestError; pub use json_object_failure::JsonObjectFailure; pub use json_schema_to_grammar_error::JsonSchemaToGrammarError; pub use key_value_xml_tags_failure::KeyValueXmlTagsFailure; -pub use kv_cache_seq_add_error::KvCacheSeqAddError; -pub use kv_cache_seq_div_error::KvCacheSeqDivError; -pub use kv_cache_seq_pos_max_error::KvCacheSeqPosMaxError; pub use llama_context_load_error::LlamaContextLoadError; pub use llama_cpp_error::LlamaCppError; pub use llama_lora_adapter_init_error::LlamaLoraAdapterInitError; pub use llama_lora_adapters_error::LlamaLoraAdaptersError; pub use llama_model_load_error::LlamaModelLoadError; pub use logits_error::LogitsError; +pub use memory_seq_add_error::MemorySeqAddError; +pub use memory_seq_cp_error::MemorySeqCpError; +pub use memory_seq_div_error::MemorySeqDivError; +pub use memory_seq_pos_max_error::MemorySeqPosMaxError; +pub use memory_seq_rm_error::MemorySeqRmError; pub use meta_val_error::MetaValError; pub use model_params_error::ModelParamsError; pub use new_llama_chat_message_error::NewLlamaChatMessageError; @@ -86,7 +85,6 @@ pub use sampling_error::SamplingError; pub use string_to_token_error::StringToTokenError; pub use token_attr_error::TokenAttrError; pub use token_outside_vocabulary::TokenOutsideVocabulary; -pub use token_sampling_error::TokenSamplingError; pub use token_to_string_error::TokenToStringError; pub use tool_call_format_failure::ToolCallFormatFailure; pub use xml_function_tags_failure::XmlFunctionTagsFailure; diff --git a/llama-cpp-bindings/src/error/kv_cache_seq_div_error.rs b/llama-cpp-bindings/src/error/memory_seq_add_error.rs similarity index 96% rename from llama-cpp-bindings/src/error/kv_cache_seq_div_error.rs rename to llama-cpp-bindings/src/error/memory_seq_add_error.rs index 463d345f0..f244bb4b9 100644 --- a/llama-cpp-bindings/src/error/kv_cache_seq_div_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_add_error.rs @@ -1,7 +1,7 @@ use std::num::TryFromIntError; #[derive(Debug, Eq, PartialEq, thiserror::Error)] -pub enum KvCacheSeqDivError { +pub enum MemorySeqAddError { #[error(transparent)] FfiStatus(#[from] crate::FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/error/copy_kv_cache_seq_error.rs b/llama-cpp-bindings/src/error/memory_seq_cp_error.rs similarity index 92% rename from llama-cpp-bindings/src/error/copy_kv_cache_seq_error.rs rename to llama-cpp-bindings/src/error/memory_seq_cp_error.rs index 011e70744..37e4a15b9 100644 --- a/llama-cpp-bindings/src/error/copy_kv_cache_seq_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_cp_error.rs @@ -1,7 +1,7 @@ use std::num::TryFromIntError; #[derive(Debug, Eq, PartialEq, thiserror::Error)] -pub enum CopyKvCacheSeqError { +pub enum MemorySeqCpError { #[error("provided start position is too large for an i32")] P0TooLarge(#[source] TryFromIntError), #[error("provided end position is too large for an i32")] diff --git a/llama-cpp-bindings/src/error/kv_cache_seq_add_error.rs b/llama-cpp-bindings/src/error/memory_seq_div_error.rs similarity index 96% rename from llama-cpp-bindings/src/error/kv_cache_seq_add_error.rs rename to llama-cpp-bindings/src/error/memory_seq_div_error.rs index a018a705f..61e22b699 100644 --- a/llama-cpp-bindings/src/error/kv_cache_seq_add_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_div_error.rs @@ -1,7 +1,7 @@ use std::num::TryFromIntError; #[derive(Debug, Eq, PartialEq, thiserror::Error)] -pub enum KvCacheSeqAddError { +pub enum MemorySeqDivError { #[error(transparent)] FfiStatus(#[from] crate::FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/error/kv_cache_seq_pos_max_error.rs b/llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs similarity index 94% rename from llama-cpp-bindings/src/error/kv_cache_seq_pos_max_error.rs rename to llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs index 0c8fbba6a..e281a077b 100644 --- a/llama-cpp-bindings/src/error/kv_cache_seq_pos_max_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs @@ -1,5 +1,5 @@ #[derive(Debug, Eq, PartialEq, thiserror::Error)] -pub enum KvCacheSeqPosMaxError { +pub enum MemorySeqPosMaxError { #[error(transparent)] FfiContract(#[from] crate::FfiContractError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/error/clear_kv_cache_seq_error.rs b/llama-cpp-bindings/src/error/memory_seq_rm_error.rs similarity index 94% rename from llama-cpp-bindings/src/error/clear_kv_cache_seq_error.rs rename to llama-cpp-bindings/src/error/memory_seq_rm_error.rs index 45c01bcdc..be9b70b8e 100644 --- a/llama-cpp-bindings/src/error/clear_kv_cache_seq_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_rm_error.rs @@ -2,7 +2,7 @@ use std::ffi::c_int; use std::num::TryFromIntError; #[derive(Debug, Eq, PartialEq, thiserror::Error)] -pub enum ClearKvCacheSeqError { +pub enum MemorySeqRmError { #[error("provided sequence id is too large for an i32")] SeqIdTooLarge(#[source] TryFromIntError), #[error("provided start position is too large for an i32")] diff --git a/llama-cpp-bindings/src/error/token_sampling_error.rs b/llama-cpp-bindings/src/error/token_sampling_error.rs deleted file mode 100644 index 993507620..000000000 --- a/llama-cpp-bindings/src/error/token_sampling_error.rs +++ /dev/null @@ -1,12 +0,0 @@ -use crate::SamplingError; -use crate::error::sampler_apply_error::SamplerApplyError; - -#[derive(Debug, Eq, PartialEq, thiserror::Error)] -pub enum TokenSamplingError { - #[error("sampler initialization failed: {0}")] - SamplerInitialization(#[from] SamplingError), - #[error("No token was selected by the sampler")] - NoTokenSelected, - #[error("applying the sampler to the token data array failed: {0}")] - SamplerApply(#[from] SamplerApplyError), -} diff --git a/llama-cpp-bindings/src/ggml_time_us.rs b/llama-cpp-bindings/src/ggml_time_us.rs deleted file mode 100644 index 4d9db374d..000000000 --- a/llama-cpp-bindings/src/ggml_time_us.rs +++ /dev/null @@ -1,21 +0,0 @@ -#[must_use] -pub fn ggml_time_us() -> i64 { - unsafe { llama_cpp_bindings_sys::ggml_time_us() } -} - -#[cfg(test)] -mod tests { - use serial_test::serial; - - use super::ggml_time_us; - use crate::llama_backend::LlamaBackend; - - #[test] - #[serial] - fn returns_positive_value() { - let _backend = LlamaBackend::init().unwrap(); - let time_microseconds = ggml_time_us(); - - assert!(time_microseconds > 0); - } -} diff --git a/llama-cpp-bindings/src/lib.rs b/llama-cpp-bindings/src/lib.rs index a59974b27..e7ec77cd6 100644 --- a/llama-cpp-bindings/src/lib.rs +++ b/llama-cpp-bindings/src/lib.rs @@ -3,6 +3,7 @@ deny(clippy::unwrap_used, clippy::expect_used, clippy::panic) )] +mod backend_device_index; pub mod bare_json_tool_call_probe; pub mod bare_json_tool_calls; pub mod batch_add_error; @@ -31,7 +32,6 @@ pub mod dry_sequence_breakers; pub mod error; pub mod eval_multimodal_chunks_params; pub mod generation_progress; -pub mod ggml_time_us; pub mod gguf_context; pub mod gguf_context_error; pub mod gguf_tensor_f32; @@ -63,8 +63,6 @@ pub mod log_options; pub mod log_record; pub mod mask_outcome; pub mod max_devices; -pub mod mlock_supported; -pub mod mmap_supported; pub mod model; pub mod mtmd; pub mod parsed_chat; @@ -81,14 +79,14 @@ pub mod token_piece; pub mod tool_call_format; pub use error::{ - ChatSessionError, ChatTemplateError, ChatTemplatesError, ClearKvCacheSeqError, - CopyKvCacheSeqError, DecodeError, EmbeddingsError, EncodeError, EvalMultimodalChunksError, - FfiContractError, FfiStatusError, GrammarError, IngestError, JsonSchemaToGrammarError, - KvCacheSeqAddError, KvCacheSeqDivError, KvCacheSeqPosMaxError, LlamaContextLoadError, - LlamaCppError, LlamaLoraAdapterInitError, LlamaLoraAdaptersError, LlamaModelLoadError, - LogitsError, MetaValError, ModelParamsError, NewLlamaChatMessageError, ParseChatMessageError, + ChatSessionError, ChatTemplateError, ChatTemplatesError, DecodeError, EmbeddingsError, + EncodeError, EvalMultimodalChunksError, FfiContractError, FfiStatusError, GrammarError, + IngestError, JsonSchemaToGrammarError, LlamaContextLoadError, LlamaCppError, + LlamaLoraAdapterInitError, LlamaLoraAdaptersError, LlamaModelLoadError, LogitsError, + MemorySeqAddError, MemorySeqCpError, MemorySeqDivError, MemorySeqPosMaxError, MemorySeqRmError, + MetaValError, ModelParamsError, NewLlamaChatMessageError, ParseChatMessageError, ProcessFailure, Result, SampleError, SamplerAcceptError, SamplingError, StringToTokenError, - TokenAttrError, TokenOutsideVocabulary, TokenSamplingError, TokenToStringError, + TokenAttrError, TokenOutsideVocabulary, TokenToStringError, }; pub use bare_json_tool_calls::BareJsonToolCalls; @@ -121,12 +119,9 @@ pub use sampled_token_classifier::SampledTokenClassifier; pub use sampled_token_section::SampledTokenSection; pub use token_piece::TokenPiece; -pub use ggml_time_us::ggml_time_us; pub use json_schema_to_grammar::json_schema_to_grammar; pub use llama_time_us::llama_time_us; pub use max_devices::max_devices; -pub use mlock_supported::mlock_supported; -pub use mmap_supported::mmap_supported; pub use log_options::LogOptions; pub use send_logs_to_log::send_logs_to_log; diff --git a/llama-cpp-bindings/src/llama_backend.rs b/llama-cpp-bindings/src/llama_backend.rs index 5ffbb7ed0..5a15be466 100644 --- a/llama-cpp-bindings/src/llama_backend.rs +++ b/llama-cpp-bindings/src/llama_backend.rs @@ -1,8 +1,6 @@ use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering::SeqCst; -use llama_cpp_bindings_sys::ggml_log_level; - use crate::LlamaCppError; use crate::llama_backend_numa_strategy::NumaStrategy; @@ -58,19 +56,6 @@ impl LlamaBackend { pub fn supports_mlock(&self) -> bool { unsafe { llama_cpp_bindings_sys::llama_supports_mlock() } } - - pub fn void_logs(&mut self) { - unsafe { - llama_cpp_bindings_sys::llama_log_set(Some(void_log), std::ptr::null_mut()); - } - } -} - -const unsafe extern "C" fn void_log( - _level: ggml_log_level, - _text: *const ::std::os::raw::c_char, - _user_data: *mut ::std::os::raw::c_void, -) { } impl Drop for LlamaBackend { @@ -87,17 +72,6 @@ mod tests { use super::LlamaBackend; use crate::LlamaCppError; - #[test] - fn void_log_callback_accepts_a_log_record() { - unsafe { - super::void_log( - llama_cpp_bindings_sys::GGML_LOG_LEVEL_INFO, - c"test".as_ptr(), - std::ptr::null_mut(), - ); - } - } - #[test] #[serial] fn init_succeeds() { diff --git a/llama-cpp-bindings/src/llguidance_sampler.rs b/llama-cpp-bindings/src/llguidance_sampler.rs index 43fd7fbb5..aec1e71e5 100644 --- a/llama-cpp-bindings/src/llguidance_sampler.rs +++ b/llama-cpp-bindings/src/llguidance_sampler.rs @@ -280,19 +280,6 @@ mod tests { ); } - #[test] - fn a_sampler_with_a_rejected_token_is_not_built() { - assert_eq!( - letter_a_sampler() - .unwrap() - .with_tokens([OUT_OF_VOCABULARY]) - .err(), - Some(SamplerAcceptError::GrammarCallbackFailed { - message: out_of_vocabulary_failure(), - }) - ); - } - #[test] fn reset_reports_a_grammar_in_an_error_state_as_a_callback_failure() { let mut sampler = letter_a_sampler().unwrap(); diff --git a/llama-cpp-bindings/src/mlock_supported.rs b/llama-cpp-bindings/src/mlock_supported.rs deleted file mode 100644 index 96bf1728b..000000000 --- a/llama-cpp-bindings/src/mlock_supported.rs +++ /dev/null @@ -1,14 +0,0 @@ -#[must_use] -pub fn mlock_supported() -> bool { - unsafe { llama_cpp_bindings_sys::llama_supports_mlock() } -} - -#[cfg(test)] -mod tests { - use super::mlock_supported; - - #[test] - fn returns_bool_without_panic() { - let _supported: bool = mlock_supported(); - } -} diff --git a/llama-cpp-bindings/src/mmap_supported.rs b/llama-cpp-bindings/src/mmap_supported.rs deleted file mode 100644 index 47ccbfe7b..000000000 --- a/llama-cpp-bindings/src/mmap_supported.rs +++ /dev/null @@ -1,14 +0,0 @@ -#[must_use] -pub fn mmap_supported() -> bool { - unsafe { llama_cpp_bindings_sys::llama_supports_mmap() } -} - -#[cfg(test)] -mod tests { - use super::mmap_supported; - - #[test] - fn returns_bool_without_panic() { - let _supported: bool = mmap_supported(); - } -} diff --git a/llama-cpp-bindings/src/model/params.rs b/llama-cpp-bindings/src/model/params.rs index 25d5ec01b..65529a1cb 100644 --- a/llama-cpp-bindings/src/model/params.rs +++ b/llama-cpp-bindings/src/model/params.rs @@ -4,6 +4,7 @@ use std::pin::Pin; use std::ptr::null; use crate::LlamaCppError; +use crate::backend_device_index::backend_device_index; use crate::context::params::LlamaContextParams; use crate::error::{FitError, ModelParamsError}; use crate::model::llama_lazy_mode::LlamaLazyMode; @@ -305,16 +306,10 @@ impl LlamaModelParams { #[must_use] pub fn devices(&self) -> Vec { - let device_count = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_count() }; - self.devices .iter() .take_while(|device| !device.is_null()) - .filter_map(|device| { - (0..device_count).position(|device_index| { - *device == unsafe { llama_cpp_bindings_sys::ggml_backend_dev_get(device_index) } - }) - }) + .filter_map(|device| backend_device_index(*device)) .collect() } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs index 2d3120407..c34fa3131 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs @@ -5,6 +5,8 @@ use std::ptr; use llama_cpp_ffi_status::read_and_free_cpp_string; +use crate::backend_device_index::backend_device_index; + use super::mtmd_context_params::MtmdContextParams; use super::mtmd_image_token_limit::MtmdImageTokenLimit; use super::mtmd_memory_usage_error::MtmdMemoryUsageError; @@ -138,18 +140,10 @@ impl MtmdMemoryUsage { /// [`LlamaModelParams::fit_params`](crate::model::params::LlamaModelParams::fit_params); /// a device past its end has no margin to grow. pub fn add_to_fit_margins(&self, margins: &mut [usize]) { - let device_count = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_count() }; - for llama_cpp_bindings_sys::llama_rs_mtmd_device_memory { device, bytes } in &self.device_memory { - let margin = (0..device_count) - .position(|device_index| { - let registered_device = - unsafe { llama_cpp_bindings_sys::ggml_backend_dev_get(device_index) }; - - registered_device == *device - }) + let margin = backend_device_index(*device) .and_then(|device_index| margins.get_mut(device_index)); if let Some(margin) = margin { diff --git a/llama-cpp-bindings/src/sampling.rs b/llama-cpp-bindings/src/sampling.rs index 4ac5e99d1..3544469cd 100644 --- a/llama-cpp-bindings/src/sampling.rs +++ b/llama-cpp-bindings/src/sampling.rs @@ -8,6 +8,7 @@ use llama_cpp_error_recorder::RecordedError; use crate::context::LlamaContext; use crate::dry_sequence_breakers::dry_sequence_breakers; +use crate::error::SamplerApplyError; use crate::model::LlamaModel; use crate::sanitized_grammar::SanitizedGrammar; use crate::token::LlamaToken; @@ -62,6 +63,53 @@ fn check_sampler_accept_status( } } +fn sampler_apply_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_sampler_apply_status, + out_error: *mut std::os::raw::c_char, +) -> Result<(), SamplerApplyError> { + match status { + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_OK => Ok(()), + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_SAMPLER_ARG => { + Err(SamplerApplyError::NullSampler) + } + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_ERROR_STRING_ALLOCATION_FAILED => { + Err(SamplerApplyError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_OUT_OF_MEMORY => { + Err(SamplerApplyError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + "llama_rs_sampler_apply", + "reported a thrown C++ exception without an error message", + ) + }?; + Err(SamplerApplyError::Reported { message }) + } + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_DATA_ARRAY_ARG => { + Err(crate::FfiContractError { + operation: "llama_rs_sampler_apply", + detail: "was given a null data_array argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_OUT_ERROR_ARG => { + Err(crate::FfiContractError { + operation: "llama_rs_sampler_apply", + detail: "was given a null out_error argument", + } + .into()) + } + other => Err(crate::FfiStatusError { + operation: "llama_rs_sampler_apply", + code: i64::from(other), + } + .into()), + } +} + fn sampler_sample_status_to_result( status: llama_cpp_bindings_sys::llama_rs_sampler_sample_status, token: i32, @@ -446,12 +494,36 @@ impl LlamaSampler { /// every candidate. pub fn apply(&self, data_array: &mut LlamaTokenDataArray) -> Result<(), SampleError> { let scope = ErrorScope::enter(); - let applied = data_array.apply_sampler(self); + let applied = self.apply_to_candidates(data_array); grammar_callback_error_to_result(scope.take())?; Ok(applied?) } + fn apply_to_candidates( + &self, + data_array: &mut LlamaTokenDataArray, + ) -> Result<(), SamplerApplyError> { + unsafe { + data_array.modify_as_c_llama_token_data_array(|c_llama_token_data_array| { + let mut out_error: *mut c_char = std::ptr::null_mut(); + let status = llama_cpp_bindings_sys::llama_rs_sampler_apply( + self.sampler.as_ptr(), + c_llama_token_data_array, + &raw mut out_error, + ); + + sampler_apply_status_to_result(status, out_error) + }) + }??; + + if data_array.has_selectable_candidate() { + Ok(()) + } else { + Err(SamplerApplyError::EveryCandidateExcluded) + } + } + /// # Errors /// Returns [`SamplerAcceptError`] if the underlying sampler rejects the token. pub fn accept(&mut self, token: LlamaToken) -> Result<(), SamplerAcceptError> { @@ -484,17 +556,6 @@ impl LlamaSampler { Ok(()) } - /// # Errors - /// Returns [`SamplerAcceptError`] if the underlying sampler rejects any token. - pub fn with_tokens( - mut self, - tokens: impl IntoIterator>, - ) -> Result { - self.accept_many(tokens)?; - - Ok(self) - } - /// # Errors /// /// Returns [`SampleError`] if the grammar sampler callback recorded a failure during reset. @@ -1158,16 +1219,6 @@ mod tests { .expect("test: accept_many should succeed"); } - #[test] - fn with_tokens_builder_pattern() { - use crate::token::LlamaToken; - - let _sampler = LlamaSampler::chain([penalties_sampler(), greedy_sampler()]) - .expect("sampler chain must initialize") - .with_tokens([LlamaToken::new(10), LlamaToken::new(20)]) - .expect("test: with_tokens should succeed"); - } - #[test] fn all_sampler_constructors() { use crate::token::LlamaToken; @@ -2126,3 +2177,223 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { ); } } + +#[cfg(test)] +mod sampler_application_tests { + use llama_cpp_fault_injection::ArmedFault; + use llama_cpp_fault_injection::injected_fault_message; + + use super::LlamaSampler; + use crate::SampleError; + use crate::SamplingError; + use crate::error::SamplerApplyError; + use crate::token::LlamaToken; + use crate::token::data::LlamaTokenData; + use crate::token::data_array::LlamaTokenDataArray; + + fn excluded_candidates() -> LlamaTokenDataArray { + LlamaTokenDataArray::new( + vec![ + LlamaTokenData::new(LlamaToken::new(0), f32::NEG_INFINITY, 0.0), + LlamaTokenData::new(LlamaToken::new(1), f32::NEG_INFINITY, 0.0), + ], + false, + ) + } + + unsafe extern "C" fn claim_more_candidates_than_exist( + _sampler: *mut llama_cpp_bindings_sys::llama_sampler, + candidates: *mut llama_cpp_bindings_sys::llama_token_data_array, + ) { + unsafe { (*candidates).size = usize::MAX }; + } + + static mut OVERREPORTING_SAMPLER: llama_cpp_bindings_sys::llama_sampler_i = + llama_cpp_bindings_sys::llama_sampler_i { + name: None, + accept: None, + apply: Some(claim_more_candidates_than_exist), + reset: None, + clone: None, + free: None, + backend_init: None, + backend_accept: None, + backend_apply: None, + backend_set_input: None, + backend_reset: None, + copy_state: None, + }; + + #[test] + fn a_sampler_claiming_more_candidates_than_exist_is_a_contract_violation() { + let overreporting_sampler = + LlamaSampler::initialize("overreporting", |out_sampler, out_error| unsafe { + llama_cpp_bindings_sys::llama_rs_sampler_init( + &raw mut OVERREPORTING_SAMPLER, + std::ptr::null_mut(), + out_sampler, + out_error, + ) + }) + .unwrap(); + + assert_eq!( + overreporting_sampler.apply(&mut excluded_candidates()), + Err(SampleError::SamplerApply(SamplerApplyError::FfiContract( + crate::FfiContractError { + operation: "modify_as_c_llama_token_data_array", + detail: "the llama.cpp sampler grew the token data array beyond its capacity", + } + ))) + ); + } + + #[test] + fn selecting_samplers_llama_cpp_cannot_build_are_reported() { + let distribution_failure = { + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_INIT_DIST_THROWS, + ); + + LlamaSampler::dist(7).err() + }; + let greedy_failure = { + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GREEDY_THROWS, + ); + + LlamaSampler::greedy().err() + }; + + assert_eq!( + [distribution_failure, greedy_failure], + [ + Some(SamplingError::Reported { + sampler: "distribution", + message: injected_fault_message(), + }), + Some(SamplingError::Reported { + sampler: "greedy", + message: injected_fault_message(), + }), + ] + ); + } +} + +#[cfg(test)] +mod sampler_apply_status_to_result_tests { + use std::ptr; + + use super::sampler_apply_status_to_result; + use crate::error::SamplerApplyError; + + #[test] + fn sampler_apply_status_allocation_failed_returns_not_enough_memory() { + assert_eq!( + sampler_apply_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_ERROR_STRING_ALLOCATION_FAILED, + ptr::null_mut(), + ), + Err(SamplerApplyError::NotEnoughMemory), + ); + } + + #[test] + fn sampler_apply_status_cxx_exception_without_a_message_is_a_contract_error() { + assert_eq!( + sampler_apply_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + ), + Err(crate::FfiContractError { + operation: "llama_rs_sampler_apply", + detail: "reported a thrown C++ exception without an error message", + } + .into()), + ); + } + + #[test] + fn sampler_apply_unknown_status_is_preserved() { + let result = sampler_apply_status_to_result(255, ptr::null_mut()); + + assert_eq!( + result, + Err(SamplerApplyError::FfiStatus(crate::FfiStatusError { + operation: "llama_rs_sampler_apply", + code: 255, + })) + ); + } + + #[test] + fn sampler_apply_status_to_result_maps_null_sampler_arg() { + let outcome = sampler_apply_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_SAMPLER_ARG, + ptr::null_mut(), + ); + assert_eq!(outcome.err(), Some(SamplerApplyError::NullSampler)); + } + + #[test] + fn sampler_apply_status_to_result_maps_null_data_array_arg() { + let outcome = sampler_apply_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_DATA_ARRAY_ARG, + ptr::null_mut(), + ); + assert_eq!( + outcome.err(), + Some( + crate::FfiContractError { + operation: "llama_rs_sampler_apply", + detail: "was given a null data_array argument", + } + .into() + ) + ); + } + + #[test] + fn sampler_apply_status_to_result_maps_null_out_error_arg() { + let outcome = sampler_apply_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_OUT_ERROR_ARG, + ptr::null_mut(), + ); + assert_eq!( + outcome.err(), + Some( + crate::FfiContractError { + operation: "llama_rs_sampler_apply", + detail: "was given a null out_error argument", + } + .into() + ) + ); + } + + #[test] + fn sampler_apply_status_to_result_maps_llama_cpp_out_of_memory() { + let outcome = sampler_apply_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_OUT_OF_MEMORY, + ptr::null_mut(), + ); + assert_eq!(outcome.err(), Some(SamplerApplyError::LlamaCppOutOfMemory)); + } + + #[test] + fn sampler_apply_status_cxx_exception_carries_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"apply threw".as_ptr()) }; + + assert_eq!( + sampler_apply_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_THREW_CXX_EXCEPTION, + out_error, + ), + Err(SamplerApplyError::Reported { + message: "apply threw".to_owned(), + }) + ); + } +} diff --git a/llama-cpp-bindings/src/token/data_array.rs b/llama-cpp-bindings/src/token/data_array.rs index 4d9b9fc16..d58adf1cc 100644 --- a/llama-cpp-bindings/src/token/data_array.rs +++ b/llama-cpp-bindings/src/token/data_array.rs @@ -1,59 +1,8 @@ use std::ptr; -use crate::error::SamplerApplyError; -use crate::error::TokenSamplingError; -use crate::sampling::LlamaSampler; use crate::token::data::LlamaTokenData; use super::LlamaToken; -use llama_cpp_ffi_status::read_and_free_cpp_string; - -fn sampler_apply_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_sampler_apply_status, - out_error: *mut std::os::raw::c_char, -) -> Result<(), SamplerApplyError> { - match status { - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_SAMPLER_ARG => { - Err(SamplerApplyError::NullSampler) - } - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_ERROR_STRING_ALLOCATION_FAILED => { - Err(SamplerApplyError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_OUT_OF_MEMORY => { - Err(SamplerApplyError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - out_error, - "llama_rs_sampler_apply", - "reported a thrown C++ exception without an error message", - ) - }?; - Err(SamplerApplyError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_DATA_ARRAY_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_sampler_apply", - detail: "was given a null data_array argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_sampler_apply", - detail: "was given a null out_error argument", - } - .into()) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_sampler_apply", - code: i64::from(other), - } - .into()), - } -} #[derive(Debug, Clone, PartialEq)] pub struct LlamaTokenDataArray { @@ -72,13 +21,6 @@ impl LlamaTokenDataArray { } } - pub fn from_iter(data: TIterator, sorted: bool) -> Self - where - TIterator: IntoIterator, - { - Self::new(data.into_iter().collect(), sorted) - } - pub fn replace_candidates(&mut self, candidates: TIterator) where TIterator: IntoIterator, @@ -93,6 +35,14 @@ impl LlamaTokenDataArray { pub fn selected_token(&self) -> Option { self.data.get(self.selected?).map(LlamaTokenData::id) } + + /// Whether a sampler left at least one candidate it could still select. + #[must_use] + pub fn has_selectable_candidate(&self) -> bool { + self.data + .iter() + .any(|candidate| candidate.logit() > f32::NEG_INFINITY) + } } impl LlamaTokenDataArray { @@ -156,319 +106,14 @@ impl LlamaTokenDataArray { Ok(result) } - - /// # Errors - /// - /// Returns [`SamplerApplyError`] if the sampler pointer is null, the llama.cpp - /// sampler runs out of memory, it throws a C++ exception while applying, or it - /// excludes every candidate so that no token is left to select. - pub fn apply_sampler(&mut self, sampler: &LlamaSampler) -> Result<(), SamplerApplyError> { - let applied = unsafe { - self.modify_as_c_llama_token_data_array(|c_llama_token_data_array| { - let mut out_error: *mut std::os::raw::c_char = ptr::null_mut(); - let status = llama_cpp_bindings_sys::llama_rs_sampler_apply( - sampler.as_ptr(), - c_llama_token_data_array, - &raw mut out_error, - ); - sampler_apply_status_to_result(status, out_error) - })? - }; - - applied.and_then(|()| self.ensure_a_candidate_remains()) - } - - fn ensure_a_candidate_remains(&self) -> Result<(), SamplerApplyError> { - if self - .data - .iter() - .any(|candidate| candidate.logit() > f32::NEG_INFINITY) - { - Ok(()) - } else { - Err(SamplerApplyError::EveryCandidateExcluded) - } - } - - /// # Errors - /// Returns [`SamplerApplyError`] if applying the sampler fails. - pub fn with_sampler(mut self, sampler: &mut LlamaSampler) -> Result { - self.apply_sampler(sampler)?; - Ok(self) - } - - /// # Errors - /// Returns [`TokenSamplingError::SamplerApply`] if applying the sampler fails, or - /// [`TokenSamplingError::NoTokenSelected`] if the sampler fails to select a token. - pub fn sample_token(&mut self, seed: u32) -> Result { - let sampler = LlamaSampler::dist(seed)?; - self.apply_sampler(&sampler)?; - self.selected_token() - .ok_or(TokenSamplingError::NoTokenSelected) - } - - /// # Errors - /// Returns [`TokenSamplingError::SamplerApply`] if applying the sampler fails, or - /// [`TokenSamplingError::NoTokenSelected`] if the sampler fails to select a token. - pub fn sample_token_greedy(&mut self) -> Result { - let sampler = LlamaSampler::greedy()?; - self.apply_sampler(&sampler)?; - self.selected_token() - .ok_or(TokenSamplingError::NoTokenSelected) - } } #[cfg(test)] mod tests { - use llama_cpp_fault_injection::ArmedFault; - use llama_cpp_fault_injection::injected_fault_message; - use super::LlamaTokenDataArray; - use crate::error::SamplerApplyError; - use crate::error::SamplingError; - use crate::error::TokenSamplingError; use crate::token::LlamaToken; use crate::token::data::LlamaTokenData; - #[test] - fn a_sampler_excluding_every_candidate_leaves_nothing_to_select() { - use crate::sampling::LlamaSampler; - use crate::token::logit_bias::LlamaLogitBias; - - let mut array = LlamaTokenDataArray::new( - vec![ - LlamaTokenData::new(LlamaToken::new(0), 1.0, 0.0), - LlamaTokenData::new(LlamaToken::new(1), 2.0, 0.0), - ], - false, - ); - let excluding_sampler = LlamaSampler::logit_bias( - 2, - &[ - LlamaLogitBias::new(LlamaToken::new(0), f32::NEG_INFINITY), - LlamaLogitBias::new(LlamaToken::new(1), f32::NEG_INFINITY), - ], - ) - .expect("the logit bias sampler must initialize"); - - assert_eq!( - array.apply_sampler(&excluding_sampler), - Err(SamplerApplyError::EveryCandidateExcluded) - ); - } - - fn excluded_candidates() -> LlamaTokenDataArray { - LlamaTokenDataArray::new( - vec![ - LlamaTokenData::new(LlamaToken::new(0), f32::NEG_INFINITY, 0.0), - LlamaTokenData::new(LlamaToken::new(1), f32::NEG_INFINITY, 0.0), - ], - false, - ) - } - - unsafe extern "C" fn claim_more_candidates_than_exist( - _sampler: *mut llama_cpp_bindings_sys::llama_sampler, - candidates: *mut llama_cpp_bindings_sys::llama_token_data_array, - ) { - unsafe { (*candidates).size = usize::MAX }; - } - - static mut OVERREPORTING_SAMPLER: llama_cpp_bindings_sys::llama_sampler_i = - llama_cpp_bindings_sys::llama_sampler_i { - name: None, - accept: None, - apply: Some(claim_more_candidates_than_exist), - reset: None, - clone: None, - free: None, - backend_init: None, - backend_accept: None, - backend_apply: None, - backend_set_input: None, - backend_reset: None, - copy_state: None, - }; - - #[test] - fn a_sampler_claiming_more_candidates_than_exist_is_a_contract_violation() { - use crate::sampling::LlamaSampler; - - let overreporting_sampler = - LlamaSampler::initialize("overreporting", |out_sampler, out_error| unsafe { - llama_cpp_bindings_sys::llama_rs_sampler_init( - &raw mut OVERREPORTING_SAMPLER, - std::ptr::null_mut(), - out_sampler, - out_error, - ) - }) - .unwrap(); - - assert_eq!( - excluded_candidates().apply_sampler(&overreporting_sampler), - Err(SamplerApplyError::FfiContract(crate::FfiContractError { - operation: "modify_as_c_llama_token_data_array", - detail: "the llama.cpp sampler grew the token data array beyond its capacity", - })) - ); - } - - #[test] - fn building_candidates_with_a_sampler_that_excludes_all_of_them_fails() { - use crate::sampling::LlamaSampler; - - assert_eq!( - excluded_candidates() - .with_sampler(&mut LlamaSampler::greedy().unwrap()) - .err(), - Some(SamplerApplyError::EveryCandidateExcluded) - ); - } - - #[test] - fn sampling_candidates_that_are_all_excluded_selects_no_token() { - assert_eq!( - [ - excluded_candidates().sample_token(7), - excluded_candidates().sample_token_greedy(), - ], - [ - Err(TokenSamplingError::SamplerApply( - SamplerApplyError::EveryCandidateExcluded - )), - Err(TokenSamplingError::SamplerApply( - SamplerApplyError::EveryCandidateExcluded - )), - ] - ); - } - - #[test] - fn sampling_reports_a_selecting_sampler_llama_cpp_cannot_build() { - let distribution_failure = { - let _fault = ArmedFault::at( - llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_INIT_DIST_THROWS, - ); - - excluded_candidates().sample_token(7) - }; - let greedy_failure = { - let _fault = ArmedFault::at( - llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GREEDY_THROWS, - ); - - excluded_candidates().sample_token_greedy() - }; - - assert_eq!( - [distribution_failure, greedy_failure], - [ - Err(TokenSamplingError::SamplerInitialization( - SamplingError::Reported { - sampler: "distribution", - message: injected_fault_message(), - } - )), - Err(TokenSamplingError::SamplerInitialization( - SamplingError::Reported { - sampler: "greedy", - message: injected_fault_message(), - } - )), - ] - ); - } - - #[test] - fn apply_greedy_sampler_selects_highest_logit() { - use crate::sampling::LlamaSampler; - - let mut array = LlamaTokenDataArray::new( - vec![ - LlamaTokenData::new(LlamaToken::new(0), 1.0, 0.0), - LlamaTokenData::new(LlamaToken::new(1), 5.0, 0.0), - LlamaTokenData::new(LlamaToken::new(2), 3.0, 0.0), - ], - false, - ); - - let sampler = LlamaSampler::greedy().expect("greedy sampler must initialize"); - array - .apply_sampler(&sampler) - .expect("test: greedy sampler must apply"); - - assert_eq!(array.selected_token(), Some(LlamaToken::new(1))); - } - - #[test] - fn with_sampler_builder_pattern() { - use crate::sampling::LlamaSampler; - - let mut sampler = LlamaSampler::greedy().expect("greedy sampler must initialize"); - let array = LlamaTokenDataArray::new( - vec![ - LlamaTokenData::new(LlamaToken::new(0), 1.0, 0.0), - LlamaTokenData::new(LlamaToken::new(1), 5.0, 0.0), - ], - false, - ) - .with_sampler(&mut sampler) - .expect("test: building with greedy sampler must succeed"); - - assert_eq!(array.selected_token(), Some(LlamaToken::new(1))); - } - - #[test] - fn sample_token_greedy_returns_highest() { - let mut array = LlamaTokenDataArray::new( - vec![ - LlamaTokenData::new(LlamaToken::new(10), 0.1, 0.0), - LlamaTokenData::new(LlamaToken::new(20), 9.9, 0.0), - ], - false, - ); - - let token = array - .sample_token_greedy() - .expect("test: greedy sampler should select a token"); - - assert_eq!(token, LlamaToken::new(20)); - } - - #[test] - fn from_iter_creates_array_from_iterator() { - let array = LlamaTokenDataArray::from_iter( - [ - LlamaTokenData::new(LlamaToken::new(0), 0.0, 0.0), - LlamaTokenData::new(LlamaToken::new(1), 1.0, 0.0), - LlamaTokenData::new(LlamaToken::new(2), 2.0, 0.0), - ], - false, - ); - - assert_eq!(array.data.len(), 3); - assert!(!array.sorted); - assert!(array.selected.is_none()); - } - - #[test] - fn sample_token_with_seed_selects_a_token() { - let mut array = LlamaTokenDataArray::new( - vec![ - LlamaTokenData::new(LlamaToken::new(10), 1.0, 0.0), - LlamaTokenData::new(LlamaToken::new(20), 1.0, 0.0), - ], - false, - ); - - let token = array - .sample_token(42) - .expect("test: dist sampler should select a token"); - - assert!(token == LlamaToken::new(10) || token == LlamaToken::new(20)); - } - #[test] fn selected_token_returns_none_when_no_selection() { let array = LlamaTokenDataArray::new( @@ -611,120 +256,3 @@ mod tests { assert_eq!(array.selected, Some(1)); } } - -#[cfg(test)] -mod sampler_apply_status_to_result_tests { - use std::ptr; - - use super::sampler_apply_status_to_result; - use crate::error::SamplerApplyError; - - #[test] - fn sampler_apply_status_allocation_failed_returns_not_enough_memory() { - assert_eq!( - sampler_apply_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - ), - Err(SamplerApplyError::NotEnoughMemory), - ); - } - - #[test] - fn sampler_apply_status_cxx_exception_without_a_message_is_a_contract_error() { - assert_eq!( - sampler_apply_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null_mut(), - ), - Err(crate::FfiContractError { - operation: "llama_rs_sampler_apply", - detail: "reported a thrown C++ exception without an error message", - } - .into()), - ); - } - - #[test] - fn sampler_apply_unknown_status_is_preserved() { - let result = sampler_apply_status_to_result(255, ptr::null_mut()); - - assert_eq!( - result, - Err(SamplerApplyError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_sampler_apply", - code: 255, - })) - ); - } - - #[test] - fn sampler_apply_status_to_result_maps_null_sampler_arg() { - let outcome = sampler_apply_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_SAMPLER_ARG, - ptr::null_mut(), - ); - assert_eq!(outcome.err(), Some(SamplerApplyError::NullSampler)); - } - - #[test] - fn sampler_apply_status_to_result_maps_null_data_array_arg() { - let outcome = sampler_apply_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_DATA_ARRAY_ARG, - ptr::null_mut(), - ); - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_sampler_apply", - detail: "was given a null data_array argument", - } - .into() - ) - ); - } - - #[test] - fn sampler_apply_status_to_result_maps_null_out_error_arg() { - let outcome = sampler_apply_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_OUT_ERROR_ARG, - ptr::null_mut(), - ); - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_sampler_apply", - detail: "was given a null out_error argument", - } - .into() - ) - ); - } - - #[test] - fn sampler_apply_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = sampler_apply_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null_mut(), - ); - assert_eq!(outcome.err(), Some(SamplerApplyError::LlamaCppOutOfMemory)); - } - - #[test] - fn sampler_apply_status_cxx_exception_carries_its_message() { - let out_error = - unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"apply threw".as_ptr()) }; - - assert_eq!( - sampler_apply_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_THREW_CXX_EXCEPTION, - out_error, - ), - Err(SamplerApplyError::Reported { - message: "apply threw".to_owned(), - }) - ); - } -} diff --git a/llama-cpp-bindings/tests/sampler_chain_without_backend.rs b/llama-cpp-bindings/tests/sampler_chain_without_backend.rs index c9175447c..4e05bd387 100644 --- a/llama-cpp-bindings/tests/sampler_chain_without_backend.rs +++ b/llama-cpp-bindings/tests/sampler_chain_without_backend.rs @@ -16,7 +16,7 @@ fn sampler_chain_applies_without_initialized_backend() -> Result<()> { false, ); - candidates.apply_sampler(&sampler_chain)?; + sampler_chain.apply(&mut candidates)?; assert_eq!(candidates.selected_token(), Some(LlamaToken::new(1))); diff --git a/llama-cpp-test-harness/src/run_to_conclusions.rs b/llama-cpp-test-harness/src/run_to_conclusions.rs index 2debc4849..fed81c05e 100644 --- a/llama-cpp-test-harness/src/run_to_conclusions.rs +++ b/llama-cpp-test-harness/src/run_to_conclusions.rs @@ -3,6 +3,8 @@ use std::sync::Arc; use libtest_mimic::Arguments; use libtest_mimic::Conclusion; use llama_cpp_bindings::llama_backend::LlamaBackend; +use llama_cpp_bindings::log_options::LogOptions; +use llama_cpp_bindings::send_logs_to_log::send_logs_to_log; use crate::execution_plan::ExecutionPlan; use crate::harness_run_error::HarnessRunError; @@ -15,12 +17,11 @@ use crate::parse_harness_arguments::parse_harness_arguments; /// the failure explicit instead of aborting the process with a panic. pub fn run_to_conclusions(arguments: Arguments) -> Result, HarnessRunError> { let arguments = parse_harness_arguments(arguments)?; - let mut backend = LlamaBackend::init()?; + let backend = Arc::new(LlamaBackend::init()?); let plan = ExecutionPlan::from_inventory(); if plan.requests_void_logs() { - backend.void_logs(); + send_logs_to_log(LogOptions::default().with_logs_enabled(false)); } - let backend = Arc::new(backend); Ok(plan.run(&backend, &arguments)) } From c5e4aee210c69152daa851587508b6174561e139 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sat, 31 Oct 2026 04:50:16 +0100 Subject: [PATCH 23/82] Import every public item from its defining module and drop the re-exports --- Cargo.lock | 2 + llama-cpp-bindings-tests/Cargo.toml | 2 + llama-cpp-bindings-tests/src/chat_answer.rs | 6 +- .../src/chat_answer_generation.rs | 6 +- .../src/chunk_token_breakdown.rs | 4 +- .../src/classify_sample_loop.rs | 2 +- .../src/classify_sample_loop_outcome.rs | 8 +- .../src/closed_reasoning_chat_session.rs | 12 +- .../src/context_after_decoding_text.rs | 6 +- .../src/ingested_generation.rs | 8 +- .../src/media_question_chat_session.rs | 4 +- .../src/paris_weather_tool_call.rs | 6 +- .../src/prime_memory_with.rs | 4 +- .../src/recognized_answer.rs | 8 +- .../tokenize_prompt_with_synthetic_image.rs | 6 +- .../src/user_turn_chat_session.rs | 10 +- .../src/weather_chat_session.rs | 10 +- .../tests/batch_context_limits.rs | 2 +- .../tests/chat_answer_parsing.rs | 10 +- .../tests/chat_prompt_rendering.rs | 10 +- .../tests/chat_session_input_validation.rs | 16 +- .../tests/chat_session_llama_cpp_failures.rs | 18 +- .../tests/chat_template_lookup.rs | 2 +- .../tests/completion_usage_accounting.rs | 18 +- .../tests/context_creation_limits.rs | 2 +- .../tests/decode_abort_control.rs | 6 +- .../tests/decode_batch_rejection.rs | 8 +- .../tests/detokenization.rs | 8 +- llama-cpp-bindings-tests/tests/dry_sampler.rs | 2 +- .../tests/duck_typed_tool_calls.rs | 2 +- .../tests/embedding_availability.rs | 4 +- .../tests/embedding_generation.rs | 8 +- .../tests/embedding_retrieval_errors.rs | 8 +- .../tests/embedding_row_width.rs | 2 +- .../tests/encode_batch_rejection.rs | 6 +- .../tests/encoder_requirement.rs | 6 +- .../end_of_generation_token_detection.rs | 2 +- .../tests/generation_end_classification.rs | 6 +- .../tests/grammar_sampler.rs | 14 +- .../tests/hidden_state_extraction.rs | 10 +- .../tests/image_question_answering.rs | 10 +- .../json_schema_constrained_generation.rs | 10 +- .../tests/lazy_grammar_sampler.rs | 6 +- .../tests/llguidance_sampler.rs | 14 +- .../tests/logits_availability.rs | 8 +- .../tests/logits_retrieval.rs | 6 +- .../tests/lora_adapters.rs | 2 +- .../tests/memory_availability.rs | 10 +- .../tests/memory_position_bounds.rs | 8 +- ...emory_position_shift_rope_compatibility.rs | 4 +- .../tests/memory_priming.rs | 2 +- .../tests/memory_sequence_edits.rs | 2 +- .../tests/memory_sequence_id_bounds.rs | 4 +- .../tests/model_loading_errors.rs | 4 +- .../tests/multimodal_audio.rs | 22 +- .../tests/multimodal_bitmap_construction.rs | 2 +- .../tests/multimodal_chunk_copy.rs | 4 +- .../tests/multimodal_chunk_encoding.rs | 6 +- .../tests/multimodal_chunk_evaluation.rs | 26 +-- .../tests/multimodal_chunk_tokenization.rs | 10 +- .../tests/multimodal_fusion.rs | 24 +-- .../multimodal_projector_capabilities.rs | 8 +- .../tests/multimodal_projector_loading.rs | 10 +- .../multimodal_prompt_parts_tokenization.rs | 20 +- .../multimodal_reasoning_classification.rs | 20 +- .../tests/multimodal_usage_accounting.rs | 16 +- .../tests/parameter_fitting.rs | 6 +- .../tests/prompt_token_accounting.rs | 4 +- .../tests/reasoning_classification.rs | 10 +- llama-cpp-bindings-tests/tests/reranking.rs | 6 +- .../tests/sampled_token_ingestion.rs | 24 +-- .../tests/session_file_token_capacity.rs | 4 +- .../tests/session_files.rs | 4 +- .../tests/special_tokens.rs | 6 +- .../tests/state_snapshots.rs | 4 +- .../tests/token_candidates.rs | 4 +- .../tests/tokenization.rs | 6 +- .../tests/tool_call_classification.rs | 8 +- .../tests/tool_call_grammar.rs | 6 +- .../tests/tool_call_parsing.rs | 18 +- .../tests/unconstrained_token_sampling.rs | 10 +- .../tests/vocabulary_bounds.rs | 6 +- llama-cpp-bindings/src/chat_classification.rs | 2 +- .../src/chat_grammar_trigger.rs | 12 +- .../src/chat_message_parse_outcome.rs | 2 +- llama-cpp-bindings/src/chat_output.rs | 2 +- llama-cpp-bindings/src/chat_session.rs | 66 +++--- llama-cpp-bindings/src/chat_span_kind.rs | 6 +- .../src/chat_template_tool_calls.rs | 2 - llama-cpp-bindings/src/chat_templates.rs | 14 +- llama-cpp-bindings/src/context.rs | 132 ++++++------ .../src/context/decoded_outputs.rs | 4 +- llama-cpp-bindings/src/context/memory.rs | 108 +++++----- llama-cpp-bindings/src/context/params.rs | 14 +- .../src/context/sequence_output_capacity.rs | 8 +- llama-cpp-bindings/src/context/session.rs | 48 +++-- .../src/context/state_data_error.rs | 4 +- .../src/dry_sequence_breakers.rs | 4 +- llama-cpp-bindings/src/error.rs | 46 +---- .../src/error/batch_transfer_error.rs | 4 +- .../src/error/chat_session_error.rs | 4 +- .../src/error/chat_templates_error.rs | 4 +- llama-cpp-bindings/src/error/decode_error.rs | 2 +- .../src/error/embeddings_error.rs | 8 +- llama-cpp-bindings/src/error/encode_error.rs | 2 +- .../src/error/eval_multimodal_chunks_error.rs | 2 +- llama-cpp-bindings/src/error/fit_error.rs | 4 +- llama-cpp-bindings/src/error/grammar_error.rs | 6 +- .../src/error/json_schema_to_grammar_error.rs | 4 +- .../src/error/llama_context_load_error.rs | 4 +- .../src/error/llama_model_load_error.rs | 4 +- .../src/error/memory_seq_add_error.rs | 4 +- .../src/error/memory_seq_div_error.rs | 4 +- .../src/error/memory_seq_pos_max_error.rs | 4 +- .../src/error/parse_chat_message_error.rs | 4 +- .../src/error/process_failure.rs | 4 +- llama-cpp-bindings/src/error/sample_error.rs | 4 +- .../src/error/sampler_accept_error.rs | 4 +- .../src/error/sampler_apply_error.rs | 4 +- .../src/error/sampling_error.rs | 4 +- .../src/error/string_to_token_error.rs | 4 +- .../src/json_schema_to_grammar.rs | 42 ++-- .../src/lazy_grammar_triggers.rs | 38 ++-- llama-cpp-bindings/src/lib.rs | 48 ----- llama-cpp-bindings/src/llama_backend.rs | 10 +- .../src/llama_backend_device.rs | 4 +- .../src/llama_backend_device_type.rs | 2 +- llama-cpp-bindings/src/llama_time_us.rs | 2 +- llama-cpp-bindings/src/llama_token_attrs.rs | 4 +- llama-cpp-bindings/src/llguidance_sampler.rs | 8 +- llama-cpp-bindings/src/log_options.rs | 2 +- llama-cpp-bindings/src/max_devices.rs | 2 +- llama-cpp-bindings/src/model.rs | 132 ++++++------ .../src/model/llama_chat_message.rs | 4 +- .../src/model/llama_chat_template.rs | 2 +- .../src/model/llama_lazy_mode.rs | 2 +- .../src/model/llama_load_mode.rs | 8 +- .../src/model/llama_lora_adapter.rs | 2 +- .../src/model/lora_adapter_scale.rs | 2 +- llama-cpp-bindings/src/model/params.rs | 77 +++---- llama-cpp-bindings/src/model/rope_type.rs | 2 +- llama-cpp-bindings/src/model/vocab_type.rs | 3 +- llama-cpp-bindings/src/mtmd.rs | 29 --- llama-cpp-bindings/src/mtmd/eval_chunk.rs | 30 +-- llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs | 82 ++++---- .../src/mtmd/mtmd_bitmap_error.rs | 4 +- llama-cpp-bindings/src/mtmd/mtmd_context.rs | 116 ++++++----- .../src/mtmd/mtmd_context_params.rs | 2 +- .../src/mtmd/mtmd_default_marker.rs | 2 +- .../src/mtmd/mtmd_encode_error.rs | 4 +- .../src/mtmd/mtmd_eval_error.rs | 4 +- .../src/mtmd/mtmd_eval_limits.rs | 2 +- .../src/mtmd/mtmd_image_token_limit.rs | 2 +- .../src/mtmd/mtmd_init_error.rs | 4 +- .../src/mtmd/mtmd_input_chunk.rs | 20 +- .../src/mtmd/mtmd_input_chunk_error.rs | 4 +- .../src/mtmd/mtmd_input_chunk_type.rs | 2 +- .../src/mtmd/mtmd_input_chunks.rs | 22 +- .../src/mtmd/mtmd_input_chunks_error.rs | 4 +- .../src/mtmd/mtmd_input_part.rs | 2 +- .../src/mtmd/mtmd_memory_usage.rs | 18 +- .../src/mtmd/mtmd_memory_usage_error.rs | 4 +- .../src/mtmd/mtmd_tokenize_error.rs | 4 +- llama-cpp-bindings/src/sampled_token.rs | 2 +- .../src/sampled_token_classifier.rs | 19 +- llama-cpp-bindings/src/sampling.rs | 191 ++++++++++-------- llama-cpp-bindings/src/send_logs_to_log.rs | 12 +- llama-cpp-bindings/src/token/data_array.rs | 8 +- llama-cpp-bindings/src/token_piece.rs | 2 +- llama-cpp-bindings/src/tool_call_format.rs | 19 +- .../src/tool_call_format/bracketed_args.rs | 4 +- .../src/tool_call_format/json_object.rs | 4 +- .../tool_call_format/key_value_xml_tags.rs | 4 +- .../src/tool_call_format/paired_quote_args.rs | 4 +- .../tool_call_format_outcome.rs | 2 +- .../src/tool_call_format/xml_function_tags.rs | 4 +- .../src/harness_run_error.rs | 2 +- llama-cpp-test-harness/src/llama_fixture.rs | 2 +- llama-cpp-test-harness/src/load_key.rs | 2 +- .../src/phase_model_files.rs | 8 +- llama-cpp-test-harness/src/phase_setup.rs | 2 +- llama-cpp-test-harness/src/phase_state.rs | 2 +- llama-cpp-test-harness/src/run.rs | 2 +- llama-cpp-test-harness/src/scheduled_phase.rs | 2 +- 184 files changed, 1133 insertions(+), 1141 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fbdabb0d7..7373be103 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1217,6 +1217,8 @@ dependencies = [ "llama-cpp-bindings-sys", "llama-cpp-bindings-types", "llama-cpp-fault-injection", + "llama-cpp-ffi-status", + "llama-cpp-gbnf", "llama-cpp-test-harness", "serde_json", "toktrie", diff --git a/llama-cpp-bindings-tests/Cargo.toml b/llama-cpp-bindings-tests/Cargo.toml index 7a0156291..ded310e2e 100644 --- a/llama-cpp-bindings-tests/Cargo.toml +++ b/llama-cpp-bindings-tests/Cargo.toml @@ -14,6 +14,8 @@ llama-cpp-bindings = { workspace = true } llama-cpp-bindings-sys = { workspace = true } llama-cpp-bindings-types = { workspace = true } llama-cpp-fault-injection = { workspace = true } +llama-cpp-ffi-status = { workspace = true } +llama-cpp-gbnf = { workspace = true } llama-cpp-test-harness = { workspace = true } serde_json = { workspace = true } toktrie = { workspace = true } diff --git a/llama-cpp-bindings-tests/src/chat_answer.rs b/llama-cpp-bindings-tests/src/chat_answer.rs index b5a80a26f..94b1677b4 100644 --- a/llama-cpp-bindings-tests/src/chat_answer.rs +++ b/llama-cpp-bindings-tests/src/chat_answer.rs @@ -1,6 +1,6 @@ -use llama_cpp_bindings::ChatOutputCompleteness; -use llama_cpp_bindings::ChatSession; -use llama_cpp_bindings::TokenUsage; +use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings_types::TokenUsage; use crate::classify_sample_loop_outcome::ClassifySampleLoopOutcome; use crate::recognized_answer::recognized_answer; diff --git a/llama-cpp-bindings-tests/src/chat_answer_generation.rs b/llama-cpp-bindings-tests/src/chat_answer_generation.rs index 74ee04c16..e1cbe795d 100644 --- a/llama-cpp-bindings-tests/src/chat_answer_generation.rs +++ b/llama-cpp-bindings-tests/src/chat_answer_generation.rs @@ -1,8 +1,8 @@ -use llama_cpp_bindings::BareJsonToolCalls; -use llama_cpp_bindings::ChatSession; -use llama_cpp_bindings::SampledTokenClassifier; +use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; +use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_test_harness::LlamaFixture; diff --git a/llama-cpp-bindings-tests/src/chunk_token_breakdown.rs b/llama-cpp-bindings-tests/src/chunk_token_breakdown.rs index f9aaabdcf..0d971141a 100644 --- a/llama-cpp-bindings-tests/src/chunk_token_breakdown.rs +++ b/llama-cpp-bindings-tests/src/chunk_token_breakdown.rs @@ -1,5 +1,5 @@ -use llama_cpp_bindings::mtmd::MtmdInputChunkType; -use llama_cpp_bindings::mtmd::MtmdInputChunks; +use llama_cpp_bindings::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::mtmd_input_chunks::MtmdInputChunks; pub struct ChunkTokenBreakdown { pub text: u64, diff --git a/llama-cpp-bindings-tests/src/classify_sample_loop.rs b/llama-cpp-bindings-tests/src/classify_sample_loop.rs index 3331ed916..16805f075 100644 --- a/llama-cpp-bindings-tests/src/classify_sample_loop.rs +++ b/llama-cpp-bindings-tests/src/classify_sample_loop.rs @@ -1,5 +1,5 @@ -use llama_cpp_bindings::GenerationProgress; use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::generation_progress::GenerationProgress; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; diff --git a/llama-cpp-bindings-tests/src/classify_sample_loop_outcome.rs b/llama-cpp-bindings-tests/src/classify_sample_loop_outcome.rs index 39c59f91d..50ac78509 100644 --- a/llama-cpp-bindings-tests/src/classify_sample_loop_outcome.rs +++ b/llama-cpp-bindings-tests/src/classify_sample_loop_outcome.rs @@ -1,7 +1,7 @@ -use llama_cpp_bindings::ChatOutput; -use llama_cpp_bindings::SampledTokenSection; +use llama_cpp_bindings::chat_output::ChatOutput; use llama_cpp_bindings::ingest_outcome::IngestOutcome; use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_bindings::sampled_token_section::SampledTokenSection; #[derive(Debug, Default)] pub struct ClassifySampleLoopOutcome { @@ -48,11 +48,11 @@ impl ClassifySampleLoopOutcome { #[cfg(test)] mod tests { - use llama_cpp_bindings::SampledTokenSection; - use llama_cpp_bindings::TokenPiece; use llama_cpp_bindings::ingest_outcome::IngestOutcome; use llama_cpp_bindings::sampled_token::SampledToken; + use llama_cpp_bindings::sampled_token_section::SampledTokenSection; use llama_cpp_bindings::token::LlamaToken; + use llama_cpp_bindings::token_piece::TokenPiece; use super::ClassifySampleLoopOutcome; diff --git a/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs b/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs index d49c7f5e8..d703c5540 100644 --- a/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs +++ b/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs @@ -1,10 +1,10 @@ -use llama_cpp_bindings::ChatAssistantPrefill; -use llama_cpp_bindings::ChatAssistantTurn; -use llama_cpp_bindings::ChatSession; -use llama_cpp_bindings::ChatSessionOptions; -use llama_cpp_bindings::ChatToolChoice; -use llama_cpp_bindings::model::LlamaChatMessage; +use llama_cpp_bindings::chat_assistant_prefill::ChatAssistantPrefill; +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; use llama_cpp_bindings::model::LlamaModel; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; const PREFILLED_REASONING: &str = "\nThe question is simple, so I answer it directly.\n"; const PREFILLED_CONTENT: &str = "\n\n"; diff --git a/llama-cpp-bindings-tests/src/context_after_decoding_text.rs b/llama-cpp-bindings-tests/src/context_after_decoding_text.rs index 64c815a12..7653c912c 100644 --- a/llama-cpp-bindings-tests/src/context_after_decoding_text.rs +++ b/llama-cpp-bindings-tests/src/context_after_decoding_text.rs @@ -1,9 +1,9 @@ use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; -use llama_cpp_bindings::context::params::LlamaPoolingType; +use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_test_harness::LlamaFixture; const TEXT: &str = "The parcel arrived on the last working day of the week."; diff --git a/llama-cpp-bindings-tests/src/ingested_generation.rs b/llama-cpp-bindings-tests/src/ingested_generation.rs index 87d8781de..cb3f3193e 100644 --- a/llama-cpp-bindings-tests/src/ingested_generation.rs +++ b/llama-cpp-bindings-tests/src/ingested_generation.rs @@ -1,9 +1,9 @@ -use llama_cpp_bindings::GenerationProgress; -use llama_cpp_bindings::SampledTokenClassifier; +use llama_cpp_bindings::generation_progress::GenerationProgress; use llama_cpp_bindings::ingest_outcome::IngestOutcome; -use llama_cpp_bindings::model::AddBos; use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; /// Feeds the tokens of `generated_text` to `classifier` as if the model generated them, then /// ends the generation with the model's end-of-sequence token. diff --git a/llama-cpp-bindings-tests/src/media_question_chat_session.rs b/llama-cpp-bindings-tests/src/media_question_chat_session.rs index 6ebfd31fd..25db5413a 100644 --- a/llama-cpp-bindings-tests/src/media_question_chat_session.rs +++ b/llama-cpp-bindings-tests/src/media_question_chat_session.rs @@ -1,7 +1,7 @@ use anyhow::Result; -use llama_cpp_bindings::ChatSession; +use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::mtmd::mtmd_default_marker; +use llama_cpp_bindings::mtmd::mtmd_default_marker::mtmd_default_marker; use crate::user_turn_chat_session::user_turn_chat_session; diff --git a/llama-cpp-bindings-tests/src/paris_weather_tool_call.rs b/llama-cpp-bindings-tests/src/paris_weather_tool_call.rs index 3afc20b8f..f44af1495 100644 --- a/llama-cpp-bindings-tests/src/paris_weather_tool_call.rs +++ b/llama-cpp-bindings-tests/src/paris_weather_tool_call.rs @@ -1,6 +1,6 @@ -use llama_cpp_bindings::ParsedToolCall; -use llama_cpp_bindings::ParsedToolCallId; -use llama_cpp_bindings::ToolCallArguments; +use llama_cpp_bindings_types::ParsedToolCall; +use llama_cpp_bindings_types::ParsedToolCallId; +use llama_cpp_bindings_types::ToolCallArguments; use serde_json::json; /// The `get_weather` call for Paris that the tool-call fixtures of every format encode. diff --git a/llama-cpp-bindings-tests/src/prime_memory_with.rs b/llama-cpp-bindings-tests/src/prime_memory_with.rs index 7893d47e9..e4f03c3bf 100644 --- a/llama-cpp-bindings-tests/src/prime_memory_with.rs +++ b/llama-cpp-bindings-tests/src/prime_memory_with.rs @@ -2,8 +2,8 @@ use anyhow::Result; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_test_harness::LlamaFixture; /// # Errors diff --git a/llama-cpp-bindings-tests/src/recognized_answer.rs b/llama-cpp-bindings-tests/src/recognized_answer.rs index f7acdac31..407b89633 100644 --- a/llama-cpp-bindings-tests/src/recognized_answer.rs +++ b/llama-cpp-bindings-tests/src/recognized_answer.rs @@ -1,7 +1,7 @@ -use llama_cpp_bindings::ChatOutput; -use llama_cpp_bindings::ChatOutputCompleteness; -use llama_cpp_bindings::ChatSession; -use llama_cpp_bindings::ParsedChatMessage; +use llama_cpp_bindings::chat_output::ChatOutput; +use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings_types::ParsedChatMessage; /// Parses `answer` with the session's parser. /// diff --git a/llama-cpp-bindings-tests/src/tokenize_prompt_with_synthetic_image.rs b/llama-cpp-bindings-tests/src/tokenize_prompt_with_synthetic_image.rs index 7eb7a3aed..d959eebe5 100644 --- a/llama-cpp-bindings-tests/src/tokenize_prompt_with_synthetic_image.rs +++ b/llama-cpp-bindings-tests/src/tokenize_prompt_with_synthetic_image.rs @@ -1,6 +1,6 @@ -use llama_cpp_bindings::mtmd::MtmdBitmap; -use llama_cpp_bindings::mtmd::MtmdInputChunks; -use llama_cpp_bindings::mtmd::MtmdInputText; +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_input_chunks::MtmdInputChunks; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; use llama_cpp_test_harness::LlamaFixture; /// Tokenizes `prompt` with a uniformly grey 64x64 image as its media. diff --git a/llama-cpp-bindings-tests/src/user_turn_chat_session.rs b/llama-cpp-bindings-tests/src/user_turn_chat_session.rs index 97b2ca2f8..050806f82 100644 --- a/llama-cpp-bindings-tests/src/user_turn_chat_session.rs +++ b/llama-cpp-bindings-tests/src/user_turn_chat_session.rs @@ -1,10 +1,10 @@ use anyhow::Result; -use llama_cpp_bindings::ChatAssistantTurn; -use llama_cpp_bindings::ChatSession; -use llama_cpp_bindings::ChatSessionOptions; -use llama_cpp_bindings::ChatToolChoice; -use llama_cpp_bindings::model::LlamaChatMessage; +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; use llama_cpp_bindings::model::LlamaModel; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; /// A chat session that answers one user turn, offering the tools of `tools_json`. /// diff --git a/llama-cpp-bindings-tests/src/weather_chat_session.rs b/llama-cpp-bindings-tests/src/weather_chat_session.rs index 33ae98d5d..5373ffdd1 100644 --- a/llama-cpp-bindings-tests/src/weather_chat_session.rs +++ b/llama-cpp-bindings-tests/src/weather_chat_session.rs @@ -1,9 +1,9 @@ -use llama_cpp_bindings::ChatAssistantTurn; -use llama_cpp_bindings::ChatSession; -use llama_cpp_bindings::ChatSessionOptions; -use llama_cpp_bindings::ChatToolChoice; -use llama_cpp_bindings::model::LlamaChatMessage; +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; use llama_cpp_bindings::model::LlamaModel; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; use crate::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; diff --git a/llama-cpp-bindings-tests/tests/batch_context_limits.rs b/llama-cpp-bindings-tests/tests/batch_context_limits.rs index 04a87a9f2..7ecbc4163 100644 --- a/llama-cpp-bindings-tests/tests/batch_context_limits.rs +++ b/llama-cpp-bindings-tests/tests/batch_context_limits.rs @@ -1,7 +1,7 @@ use anyhow::Result; -use llama_cpp_bindings::DecodeError; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::error::batch_transfer_error::BatchTransferError; +use llama_cpp_bindings::error::decode_error::DecodeError; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::token::LlamaToken; diff --git a/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs b/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs index 6f350c7ac..3399fd0c3 100644 --- a/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs +++ b/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs @@ -1,10 +1,10 @@ use anyhow::Result; use anyhow::bail; -use llama_cpp_bindings::ChatMessageParseOutcome; -use llama_cpp_bindings::ChatOutput; -use llama_cpp_bindings::ChatOutputCompleteness; -use llama_cpp_bindings::ChatToolChoice; -use llama_cpp_bindings::ParseChatMessageError; +use llama_cpp_bindings::chat_message_parse_outcome::ChatMessageParseOutcome; +use llama_cpp_bindings::chat_output::ChatOutput; +use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; +use llama_cpp_bindings::error::parse_chat_message_error::ParseChatMessageError; use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; use llama_cpp_bindings_tests::weather_chat_session::weather_chat_session; diff --git a/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs b/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs index 7b05c55f9..3cec25668 100644 --- a/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs +++ b/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs @@ -1,9 +1,9 @@ use anyhow::Result; -use llama_cpp_bindings::ChatAssistantTurn; -use llama_cpp_bindings::ChatSession; -use llama_cpp_bindings::ChatSessionOptions; -use llama_cpp_bindings::ChatToolChoice; -use llama_cpp_bindings::model::LlamaChatMessage; +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs index 20113e3dc..f3f1c0a50 100644 --- a/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs +++ b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs @@ -2,14 +2,14 @@ use std::ffi::CString; use anyhow::Result; use anyhow::bail; -use llama_cpp_bindings::ChatAssistantTurn; -use llama_cpp_bindings::ChatSession; -use llama_cpp_bindings::ChatSessionError; -use llama_cpp_bindings::ChatSessionOptions; -use llama_cpp_bindings::ChatToolChoice; -use llama_cpp_bindings::NewLlamaChatMessageError; -use llama_cpp_bindings::StringToTokenError; -use llama_cpp_bindings::model::LlamaChatMessage; +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; +use llama_cpp_bindings::error::chat_session_error::ChatSessionError; +use llama_cpp_bindings::error::new_llama_chat_message_error::NewLlamaChatMessageError; +use llama_cpp_bindings::error::string_to_token_error::StringToTokenError; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; use llama_cpp_bindings_tests::media_question_chat_session::media_question_chat_session; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; use llama_cpp_test_harness::LlamaFixture; diff --git a/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs b/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs index f2ab693ba..503940a04 100644 --- a/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs +++ b/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs @@ -1,18 +1,18 @@ use std::str::Utf8Error; -use llama_cpp_bindings::ChatOutput; -use llama_cpp_bindings::ChatOutputCompleteness; -use llama_cpp_bindings::ChatSessionError; -use llama_cpp_bindings::ChatTemplatesError; -use llama_cpp_bindings::ChatToolChoice; -use llama_cpp_bindings::FfiStatusError; -use llama_cpp_bindings::GrammarError; -use llama_cpp_bindings::ParseChatMessageError; -use llama_cpp_bindings::SamplerAcceptError; +use llama_cpp_bindings::chat_output::ChatOutput; +use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; +use llama_cpp_bindings::error::chat_session_error::ChatSessionError; +use llama_cpp_bindings::error::chat_templates_error::ChatTemplatesError; +use llama_cpp_bindings::error::grammar_error::GrammarError; +use llama_cpp_bindings::error::parse_chat_message_error::ParseChatMessageError; +use llama_cpp_bindings::error::sampler_accept_error::SamplerAcceptError; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; use llama_cpp_bindings_tests::weather_chat_session::weather_chat_session; use llama_cpp_fault_injection::ArmedFault; use llama_cpp_fault_injection::injected_fault_message; +use llama_cpp_ffi_status::FfiStatusError; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/chat_template_lookup.rs b/llama-cpp-bindings-tests/tests/chat_template_lookup.rs index 7102335a6..a53042961 100644 --- a/llama-cpp-bindings-tests/tests/chat_template_lookup.rs +++ b/llama-cpp-bindings-tests/tests/chat_template_lookup.rs @@ -1,6 +1,6 @@ use std::ffi::CString; -use llama_cpp_bindings::ChatTemplateError; +use llama_cpp_bindings::error::chat_template_error::ChatTemplateError; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs index 9b21257d3..b135f27c9 100644 --- a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs +++ b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs @@ -1,17 +1,17 @@ use anyhow::Context as _; use anyhow::Result; -use llama_cpp_bindings::BareJsonToolCalls; -use llama_cpp_bindings::ChatAssistantTurn; -use llama_cpp_bindings::ChatSession; -use llama_cpp_bindings::ChatSessionOptions; -use llama_cpp_bindings::ChatToolChoice; -use llama_cpp_bindings::SampledTokenClassifier; +use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::LlamaChatMessage; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; use llama_cpp_test_harness::LlamaFixture; diff --git a/llama-cpp-bindings-tests/tests/context_creation_limits.rs b/llama-cpp-bindings-tests/tests/context_creation_limits.rs index 0cd4b4b00..3453282bf 100644 --- a/llama-cpp-bindings-tests/tests/context_creation_limits.rs +++ b/llama-cpp-bindings-tests/tests/context_creation_limits.rs @@ -1,7 +1,7 @@ use std::num::NonZeroU32; -use llama_cpp_bindings::LlamaContextLoadError; use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::error::llama_context_load_error::LlamaContextLoadError; use llama_cpp_fault_injection::ArmedFault; use llama_cpp_fault_injection::injected_fault_message; use llama_cpp_test_harness::LlamaFixture; diff --git a/llama-cpp-bindings-tests/tests/decode_abort_control.rs b/llama-cpp-bindings-tests/tests/decode_abort_control.rs index 228a6412d..295759eda 100644 --- a/llama-cpp-bindings-tests/tests/decode_abort_control.rs +++ b/llama-cpp-bindings-tests/tests/decode_abort_control.rs @@ -2,12 +2,12 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; use anyhow::Result; -use llama_cpp_bindings::DecodeError; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::error::decode_error::DecodeError; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs b/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs index 6fe83851f..0d309fcb3 100644 --- a/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs +++ b/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs @@ -1,12 +1,12 @@ use anyhow::Result; -use llama_cpp_bindings::DecodeError; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; -use llama_cpp_bindings::context::params::LlamaAttentionType; +use llama_cpp_bindings::context::llama_attention_type::LlamaAttentionType; use llama_cpp_bindings::error::batch_transfer_error::BatchTransferError; +use llama_cpp_bindings::error::decode_error::DecodeError; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_fault_injection::ArmedFault; use llama_cpp_fault_injection::injected_fault_message; use llama_cpp_test_harness::LlamaFixture; diff --git a/llama-cpp-bindings-tests/tests/detokenization.rs b/llama-cpp-bindings-tests/tests/detokenization.rs index 9751b2d2d..d54bae55e 100644 --- a/llama-cpp-bindings-tests/tests/detokenization.rs +++ b/llama-cpp-bindings-tests/tests/detokenization.rs @@ -1,10 +1,10 @@ use std::num::NonZeroU16; use anyhow::Result; -use llama_cpp_bindings::SampledToken; -use llama_cpp_bindings::TokenToStringError; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::error::token_to_string_error::TokenToStringError; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::token::LlamaToken; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/dry_sampler.rs b/llama-cpp-bindings-tests/tests/dry_sampler.rs index 224e57f19..59399cdbe 100644 --- a/llama-cpp-bindings-tests/tests/dry_sampler.rs +++ b/llama-cpp-bindings-tests/tests/dry_sampler.rs @@ -1,6 +1,6 @@ use std::ffi::CString; -use llama_cpp_bindings::SamplingError; +use llama_cpp_bindings::error::sampling_error::SamplingError; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs b/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs index 7472cd814..c32386e52 100644 --- a/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs +++ b/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use llama_cpp_bindings::ChatOutputCompleteness; +use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; use llama_cpp_bindings_tests::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; use llama_cpp_bindings_tests::paris_weather_tool_call::paris_weather_tool_call; use llama_cpp_bindings_tests::recognized_answer::recognized_answer; diff --git a/llama-cpp-bindings-tests/tests/embedding_availability.rs b/llama-cpp-bindings-tests/tests/embedding_availability.rs index 6639fe796..6c2c0b64c 100644 --- a/llama-cpp-bindings-tests/tests/embedding_availability.rs +++ b/llama-cpp-bindings-tests/tests/embedding_availability.rs @@ -1,5 +1,5 @@ -use llama_cpp_bindings::EmbeddingsError; -use llama_cpp_bindings::context::params::LlamaPoolingType; +use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; +use llama_cpp_bindings::error::embeddings_error::EmbeddingsError; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/embedding_generation.rs b/llama-cpp-bindings-tests/tests/embedding_generation.rs index 9a57c24e2..f8d444980 100644 --- a/llama-cpp-bindings-tests/tests/embedding_generation.rs +++ b/llama-cpp-bindings-tests/tests/embedding_generation.rs @@ -1,12 +1,12 @@ use anyhow::Context; use anyhow::Result; -use llama_cpp_bindings::SampledTokenClassifier; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; -use llama_cpp_bindings::context::params::LlamaPoolingType; +use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings_tests::normalize::normalize; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs b/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs index d4408d167..7c28b52be 100644 --- a/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs +++ b/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs @@ -1,11 +1,11 @@ use anyhow::Result; -use llama_cpp_bindings::EmbeddingsError; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; -use llama_cpp_bindings::context::params::LlamaPoolingType; +use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; +use llama_cpp_bindings::error::embeddings_error::EmbeddingsError; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/embedding_row_width.rs b/llama-cpp-bindings-tests/tests/embedding_row_width.rs index 49d3ccb78..10f50b7f6 100644 --- a/llama-cpp-bindings-tests/tests/embedding_row_width.rs +++ b/llama-cpp-bindings-tests/tests/embedding_row_width.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use llama_cpp_bindings::context::params::LlamaPoolingType; +use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs b/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs index f3fb90f85..d1ac0dbe3 100644 --- a/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs +++ b/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs @@ -1,10 +1,10 @@ use anyhow::Result; -use llama_cpp_bindings::EncodeError; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::error::batch_transfer_error::BatchTransferError; +use llama_cpp_bindings::error::encode_error::EncodeError; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_fault_injection::ArmedFault; use llama_cpp_fault_injection::injected_fault_message; use llama_cpp_test_harness::LlamaFixture; diff --git a/llama-cpp-bindings-tests/tests/encoder_requirement.rs b/llama-cpp-bindings-tests/tests/encoder_requirement.rs index 7ea864c00..25d285cae 100644 --- a/llama-cpp-bindings-tests/tests/encoder_requirement.rs +++ b/llama-cpp-bindings-tests/tests/encoder_requirement.rs @@ -1,10 +1,10 @@ use anyhow::Result; -use llama_cpp_bindings::EncodeError; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::error::encode_error::EncodeError; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs b/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs index 23e1f63c3..17d5dbf63 100644 --- a/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs +++ b/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs @@ -1,6 +1,6 @@ use anyhow::Context; use anyhow::Result; -use llama_cpp_bindings::SampledToken; +use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/generation_end_classification.rs b/llama-cpp-bindings-tests/tests/generation_end_classification.rs index 19ea87c25..a6b4a5667 100644 --- a/llama-cpp-bindings-tests/tests/generation_end_classification.rs +++ b/llama-cpp-bindings-tests/tests/generation_end_classification.rs @@ -1,8 +1,8 @@ use anyhow::Result; -use llama_cpp_bindings::BareJsonToolCalls; -use llama_cpp_bindings::SampledToken; -use llama_cpp_bindings::SampledTokenClassifier; +use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::ingest_outcome::IngestOutcome; +use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings_tests::ingested_generation::ingested_generation; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; use llama_cpp_test_harness::LlamaFixture; diff --git a/llama-cpp-bindings-tests/tests/grammar_sampler.rs b/llama-cpp-bindings-tests/tests/grammar_sampler.rs index c4e42f566..d223c55b1 100644 --- a/llama-cpp-bindings-tests/tests/grammar_sampler.rs +++ b/llama-cpp-bindings-tests/tests/grammar_sampler.rs @@ -3,20 +3,20 @@ use std::ffi::CString; use anyhow::Context as _; use anyhow::Result; use anyhow::bail; -use llama_cpp_bindings::BareJsonToolCalls; -use llama_cpp_bindings::GbnfValidationError; -use llama_cpp_bindings::GrammarError; -use llama_cpp_bindings::SampledToken; -use llama_cpp_bindings::SampledTokenClassifier; +use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::error::grammar_error::GrammarError; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; use llama_cpp_fault_injection::ArmedFault; +use llama_cpp_gbnf::gbnf_validation_error::GbnfValidationError; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs b/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs index 7b7623c4d..e1abff933 100644 --- a/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs +++ b/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs @@ -1,12 +1,12 @@ use anyhow::Result; -use llama_cpp_bindings::EmbeddingsError; -use llama_cpp_bindings::SampledToken; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; -use llama_cpp_bindings::context::params::LlamaPoolingType; +use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; +use llama_cpp_bindings::error::embeddings_error::EmbeddingsError; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::token::LlamaToken; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/image_question_answering.rs b/llama-cpp-bindings-tests/tests/image_question_answering.rs index a8d53deb2..3df35469a 100644 --- a/llama-cpp-bindings-tests/tests/image_question_answering.rs +++ b/llama-cpp-bindings-tests/tests/image_question_answering.rs @@ -1,13 +1,13 @@ use anyhow::Context; use anyhow::Result; -use llama_cpp_bindings::BareJsonToolCalls; -use llama_cpp_bindings::EvalMultimodalChunksParams; +use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::eval_multimodal_chunks_params::EvalMultimodalChunksParams; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::mtmd::MtmdBitmap; -use llama_cpp_bindings::mtmd::MtmdCapability; -use llama_cpp_bindings::mtmd::MtmdInputText; +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_capability::MtmdCapability; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; diff --git a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs index 5cb243d59..95818a80b 100644 --- a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs +++ b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs @@ -1,13 +1,13 @@ use anyhow::Context as _; use anyhow::Result; -use llama_cpp_bindings::SampledToken; -use llama_cpp_bindings::SampledTokenClassifier; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; -use llama_cpp_bindings::json_schema_to_grammar; +use llama_cpp_bindings::json_schema_to_grammar::json_schema_to_grammar; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs b/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs index a102fd17a..c0d1b9df6 100644 --- a/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs +++ b/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs @@ -1,9 +1,9 @@ use std::ffi::CString; use anyhow::Result; -use llama_cpp_bindings::GrammarError; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::error::grammar_error::GrammarError; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs index 76609d201..7230be30a 100644 --- a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs +++ b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs @@ -2,19 +2,19 @@ use std::ffi::CStr; use anyhow::Context as _; use anyhow::Result; -use llama_cpp_bindings::GrammarError; -use llama_cpp_bindings::SampleError; -use llama_cpp_bindings::SamplerAcceptError; -use llama_cpp_bindings::TokenToStringError; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; -use llama_cpp_bindings::error::GrammarRuntimeError; +use llama_cpp_bindings::error::grammar_error::GrammarError; +use llama_cpp_bindings::error::grammar_runtime_error::GrammarRuntimeError; +use llama_cpp_bindings::error::sample_error::SampleError; +use llama_cpp_bindings::error::sampler_accept_error::SamplerAcceptError; +use llama_cpp_bindings::error::token_to_string_error::TokenToStringError; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::llguidance_sampler::create_llg_sampler; -use llama_cpp_bindings::model::AddBos; use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::params::LlamaModelParams; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings::token::LlamaToken; use llama_cpp_fault_injection::ArmedFault; diff --git a/llama-cpp-bindings-tests/tests/logits_availability.rs b/llama-cpp-bindings-tests/tests/logits_availability.rs index c267ed02b..2a1dfda79 100644 --- a/llama-cpp-bindings-tests/tests/logits_availability.rs +++ b/llama-cpp-bindings-tests/tests/logits_availability.rs @@ -1,8 +1,8 @@ use anyhow::Result; -use llama_cpp_bindings::LogitsError; -use llama_cpp_bindings::SampleError; -use llama_cpp_bindings::SampledTokenClassifier; -use llama_cpp_bindings::context::params::LlamaPoolingType; +use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; +use llama_cpp_bindings::error::logits_error::LogitsError; +use llama_cpp_bindings::error::sample_error::SampleError; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings::token::data_array::LlamaTokenDataArray; use llama_cpp_test_harness::LlamaFixture; diff --git a/llama-cpp-bindings-tests/tests/logits_retrieval.rs b/llama-cpp-bindings-tests/tests/logits_retrieval.rs index a08ace11a..c1710360b 100644 --- a/llama-cpp-bindings-tests/tests/logits_retrieval.rs +++ b/llama-cpp-bindings-tests/tests/logits_retrieval.rs @@ -1,10 +1,10 @@ use anyhow::Result; -use llama_cpp_bindings::LogitsError; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::error::logits_error::LogitsError; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_fault_injection::ArmedFault; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/lora_adapters.rs b/llama-cpp-bindings-tests/tests/lora_adapters.rs index d180b1d8b..ec3d6bfb9 100644 --- a/llama-cpp-bindings-tests/tests/lora_adapters.rs +++ b/llama-cpp-bindings-tests/tests/lora_adapters.rs @@ -1,6 +1,6 @@ use anyhow::Result; use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_bindings::error::LlamaLoraAdaptersError; +use llama_cpp_bindings::error::llama_lora_adapters_error::LlamaLoraAdaptersError; use llama_cpp_bindings::model::lora_adapter_scale::LoraAdapterScale; use llama_cpp_fault_injection::ArmedFault; use llama_cpp_test_harness::LlamaFixture; diff --git a/llama-cpp-bindings-tests/tests/memory_availability.rs b/llama-cpp-bindings-tests/tests/memory_availability.rs index 36282336d..6ca886b7b 100644 --- a/llama-cpp-bindings-tests/tests/memory_availability.rs +++ b/llama-cpp-bindings-tests/tests/memory_availability.rs @@ -2,11 +2,11 @@ use std::num::NonZeroU8; use anyhow::Context as _; use anyhow::Result; -use llama_cpp_bindings::MemorySeqAddError; -use llama_cpp_bindings::MemorySeqCpError; -use llama_cpp_bindings::MemorySeqDivError; -use llama_cpp_bindings::MemorySeqPosMaxError; -use llama_cpp_bindings::MemorySeqRmError; +use llama_cpp_bindings::error::memory_seq_add_error::MemorySeqAddError; +use llama_cpp_bindings::error::memory_seq_cp_error::MemorySeqCpError; +use llama_cpp_bindings::error::memory_seq_div_error::MemorySeqDivError; +use llama_cpp_bindings::error::memory_seq_pos_max_error::MemorySeqPosMaxError; +use llama_cpp_bindings::error::memory_seq_rm_error::MemorySeqRmError; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/memory_position_bounds.rs b/llama-cpp-bindings-tests/tests/memory_position_bounds.rs index fd46f7c46..5cc2e5d69 100644 --- a/llama-cpp-bindings-tests/tests/memory_position_bounds.rs +++ b/llama-cpp-bindings-tests/tests/memory_position_bounds.rs @@ -2,10 +2,10 @@ use std::num::NonZeroU8; use anyhow::Context as _; use anyhow::Result; -use llama_cpp_bindings::error::MemorySeqAddError; -use llama_cpp_bindings::error::MemorySeqCpError; -use llama_cpp_bindings::error::MemorySeqDivError; -use llama_cpp_bindings::error::MemorySeqRmError; +use llama_cpp_bindings::error::memory_seq_add_error::MemorySeqAddError; +use llama_cpp_bindings::error::memory_seq_cp_error::MemorySeqCpError; +use llama_cpp_bindings::error::memory_seq_div_error::MemorySeqDivError; +use llama_cpp_bindings::error::memory_seq_rm_error::MemorySeqRmError; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs b/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs index ad5b3b467..9ce951ce3 100644 --- a/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs +++ b/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs @@ -2,8 +2,8 @@ use std::num::NonZeroU8; use anyhow::Context as _; use anyhow::Result; -use llama_cpp_bindings::error::MemorySeqAddError; -use llama_cpp_bindings::error::MemorySeqDivError; +use llama_cpp_bindings::error::memory_seq_add_error::MemorySeqAddError; +use llama_cpp_bindings::error::memory_seq_div_error::MemorySeqDivError; use llama_cpp_bindings_tests::prime_memory::prime_memory; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/memory_priming.rs b/llama-cpp-bindings-tests/tests/memory_priming.rs index 071e224f5..e62104f4f 100644 --- a/llama-cpp-bindings-tests/tests/memory_priming.rs +++ b/llama-cpp-bindings-tests/tests/memory_priming.rs @@ -1,6 +1,6 @@ use anyhow::Result; -use llama_cpp_bindings::StringToTokenError; use llama_cpp_bindings::batch_add_error::BatchAddError; +use llama_cpp_bindings::error::string_to_token_error::StringToTokenError; use llama_cpp_bindings_tests::prime_memory_with::prime_memory_with; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs b/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs index e7dc44fd0..4c2d5c245 100644 --- a/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs +++ b/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use llama_cpp_bindings::MemorySeqRmError; +use llama_cpp_bindings::error::memory_seq_rm_error::MemorySeqRmError; use llama_cpp_bindings_tests::prime_memory::prime_memory; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs b/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs index 8350e08d1..f69d2cc93 100644 --- a/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs +++ b/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use llama_cpp_bindings::error::MemorySeqRmError; +use llama_cpp_bindings::error::memory_seq_rm_error::MemorySeqRmError; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -42,7 +42,7 @@ fn memory_seq_pos_max_rejects_out_of_range_sequence(fixture: &LlamaFixture<'_>) assert_eq!( result, - Err(llama_cpp_bindings::MemorySeqPosMaxError::SequenceIdOutOfRange { seq_id: 999 }) + Err(llama_cpp_bindings::error::memory_seq_pos_max_error::MemorySeqPosMaxError::SequenceIdOutOfRange { seq_id: 999 }) ); Ok(()) diff --git a/llama-cpp-bindings-tests/tests/model_loading_errors.rs b/llama-cpp-bindings-tests/tests/model_loading_errors.rs index 134dfee24..0f7706962 100644 --- a/llama-cpp-bindings-tests/tests/model_loading_errors.rs +++ b/llama-cpp-bindings-tests/tests/model_loading_errors.rs @@ -3,8 +3,8 @@ use std::path::Path; use std::path::PathBuf; use anyhow::Result; -use llama_cpp_bindings::LlamaLoraAdapterInitError; -use llama_cpp_bindings::LlamaModelLoadError; +use llama_cpp_bindings::error::llama_lora_adapter_init_error::LlamaLoraAdapterInitError; +use llama_cpp_bindings::error::llama_model_load_error::LlamaModelLoadError; use llama_cpp_bindings::model::LlamaModel; use llama_cpp_bindings::model::params::LlamaModelParams; use llama_cpp_test_harness::LlamaFixture; diff --git a/llama-cpp-bindings-tests/tests/multimodal_audio.rs b/llama-cpp-bindings-tests/tests/multimodal_audio.rs index 27aadca42..2d648e575 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_audio.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_audio.rs @@ -1,19 +1,19 @@ use anyhow::Context; use anyhow::Result; -use llama_cpp_bindings::BareJsonToolCalls; -use llama_cpp_bindings::ChatAssistantTurn; -use llama_cpp_bindings::ChatSession; -use llama_cpp_bindings::ChatSessionOptions; -use llama_cpp_bindings::ChatToolChoice; -use llama_cpp_bindings::EvalMultimodalChunksParams; +use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::eval_multimodal_chunks_params::EvalMultimodalChunksParams; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::LlamaChatMessage; -use llama_cpp_bindings::mtmd::MtmdBitmap; -use llama_cpp_bindings::mtmd::MtmdCapability; -use llama_cpp_bindings::mtmd::MtmdInputText; -use llama_cpp_bindings::mtmd::mtmd_default_marker; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_capability::MtmdCapability; +use llama_cpp_bindings::mtmd::mtmd_default_marker::mtmd_default_marker; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; diff --git a/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs b/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs index 071ae6e84..ffe36334f 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use llama_cpp_bindings::mtmd::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs index 8e426de92..bbb5ed2fd 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs @@ -1,7 +1,7 @@ use anyhow::Context as _; use anyhow::Result; -use llama_cpp_bindings::mtmd::MtmdBitmap; -use llama_cpp_bindings::mtmd::MtmdInputText; +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs index f41de20a7..a7270b48d 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs @@ -1,8 +1,8 @@ use anyhow::Context as _; use anyhow::Result; -use llama_cpp_bindings::mtmd::MtmdBitmap; -use llama_cpp_bindings::mtmd::MtmdInputChunkType; -use llama_cpp_bindings::mtmd::MtmdInputText; +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs index d8af7d50e..2adb7c53b 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs @@ -1,21 +1,20 @@ use anyhow::Context; use anyhow::Result; use anyhow::bail; -use llama_cpp_bindings::EvalMultimodalChunksParams; -use llama_cpp_bindings::FfiContractError; use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_bindings::error::EvalMultimodalChunksError; +use llama_cpp_bindings::error::eval_multimodal_chunks_error::EvalMultimodalChunksError; use llama_cpp_bindings::error::sequence_id_out_of_range::SequenceIdOutOfRange; -use llama_cpp_bindings::mtmd::FittedMtmdInputChunk; -use llama_cpp_bindings::mtmd::FittedMtmdInputChunks; -use llama_cpp_bindings::mtmd::MtmdBitmap; -use llama_cpp_bindings::mtmd::MtmdContext; -use llama_cpp_bindings::mtmd::MtmdEvalError; -use llama_cpp_bindings::mtmd::MtmdInputChunkType; -use llama_cpp_bindings::mtmd::MtmdInputChunkTypeError; -use llama_cpp_bindings::mtmd::MtmdInputChunks; -use llama_cpp_bindings::mtmd::MtmdInputText; -use llama_cpp_bindings::mtmd::NonCausalChunkMicroBatchMismatch; +use llama_cpp_bindings::eval_multimodal_chunks_params::EvalMultimodalChunksParams; +use llama_cpp_bindings::mtmd::fitted_mtmd_input_chunk::FittedMtmdInputChunk; +use llama_cpp_bindings::mtmd::fitted_mtmd_input_chunks::FittedMtmdInputChunks; +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_context::MtmdContext; +use llama_cpp_bindings::mtmd::mtmd_eval_error::MtmdEvalError; +use llama_cpp_bindings::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::mtmd_input_chunk_type_error::MtmdInputChunkTypeError; +use llama_cpp_bindings::mtmd::mtmd_input_chunks::MtmdInputChunks; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; +use llama_cpp_bindings::mtmd::non_causal_chunk_micro_batch_mismatch::NonCausalChunkMicroBatchMismatch; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; @@ -24,6 +23,7 @@ use llama_cpp_bindings_tests::prime_memory_with::prime_memory_with; use llama_cpp_bindings_tests::prompt_question::PROMPT_QUESTION; use llama_cpp_fault_injection::ArmedFault; use llama_cpp_fault_injection::injected_fault_message; +use llama_cpp_ffi_status::FfiContractError; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs index 6423a522f..817c51c22 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs @@ -1,10 +1,10 @@ use anyhow::Context as _; use anyhow::Result; -use llama_cpp_bindings::mtmd::MtmdBitmap; -use llama_cpp_bindings::mtmd::MtmdInputChunkType; -use llama_cpp_bindings::mtmd::MtmdInputChunksError; -use llama_cpp_bindings::mtmd::MtmdInputText; -use llama_cpp_bindings::mtmd::MtmdTokenizeError; +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::mtmd_input_chunks_error::MtmdInputChunksError; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; +use llama_cpp_bindings::mtmd::mtmd_tokenize_error::MtmdTokenizeError; use llama_cpp_bindings::token::LlamaToken; use llama_cpp_fault_injection::ArmedFault; use llama_cpp_fault_injection::injected_fault_message; diff --git a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs index e370b61c1..0f5866e0f 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs @@ -1,21 +1,21 @@ use anyhow::Context; use anyhow::Result; -use llama_cpp_bindings::BareJsonToolCalls; -use llama_cpp_bindings::ChatAssistantTurn; -use llama_cpp_bindings::ChatSession; -use llama_cpp_bindings::ChatSessionOptions; -use llama_cpp_bindings::ChatToolChoice; -use llama_cpp_bindings::EvalMultimodalChunksParams; +use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::eval_multimodal_chunks_params::EvalMultimodalChunksParams; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::LlamaChatMessage; use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::mtmd::MtmdBitmap; -use llama_cpp_bindings::mtmd::MtmdCapability; -use llama_cpp_bindings::mtmd::MtmdContext; -use llama_cpp_bindings::mtmd::MtmdInputText; -use llama_cpp_bindings::mtmd::mtmd_default_marker; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_capability::MtmdCapability; +use llama_cpp_bindings::mtmd::mtmd_context::MtmdContext; +use llama_cpp_bindings::mtmd::mtmd_default_marker::mtmd_default_marker; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; diff --git a/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs b/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs index ad7784a0f..a9b27c69c 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs @@ -1,10 +1,10 @@ use anyhow::Context as _; use anyhow::Result; use anyhow::bail; -use llama_cpp_bindings::mtmd::MtmdBitmap; -use llama_cpp_bindings::mtmd::MtmdCapability; -use llama_cpp_bindings::mtmd::MtmdInputChunkType; -use llama_cpp_bindings::mtmd::MtmdInputText; +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_capability::MtmdCapability; +use llama_cpp_bindings::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs b/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs index b93d6fa12..18eded624 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs @@ -1,11 +1,11 @@ use std::ffi::CString; use std::num::NonZeroU32; -use llama_cpp_bindings::mtmd::MtmdContext; -use llama_cpp_bindings::mtmd::MtmdContextParams; -use llama_cpp_bindings::mtmd::MtmdContextParamsError; -use llama_cpp_bindings::mtmd::MtmdImageTokenLimit; -use llama_cpp_bindings::mtmd::MtmdInitError; +use llama_cpp_bindings::mtmd::mtmd_context::MtmdContext; +use llama_cpp_bindings::mtmd::mtmd_context_params::MtmdContextParams; +use llama_cpp_bindings::mtmd::mtmd_context_params_error::MtmdContextParamsError; +use llama_cpp_bindings::mtmd::mtmd_image_token_limit::MtmdImageTokenLimit; +use llama_cpp_bindings::mtmd::mtmd_init_error::MtmdInitError; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs b/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs index 88ac48d88..f96b325d0 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs @@ -2,16 +2,16 @@ use std::ffi::CString; use anyhow::Context as _; use anyhow::Result; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; -use llama_cpp_bindings::mtmd::MtmdBitmap; -use llama_cpp_bindings::mtmd::MtmdInputChunkType; -use llama_cpp_bindings::mtmd::MtmdInputChunks; -use llama_cpp_bindings::mtmd::MtmdInputChunksError; -use llama_cpp_bindings::mtmd::MtmdInputPart; -use llama_cpp_bindings::mtmd::MtmdInputTextPart; -use llama_cpp_bindings::mtmd::MtmdTokenizeError; -use llama_cpp_bindings::mtmd::mtmd_default_marker; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_default_marker::mtmd_default_marker; +use llama_cpp_bindings::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::mtmd_input_chunks::MtmdInputChunks; +use llama_cpp_bindings::mtmd::mtmd_input_chunks_error::MtmdInputChunksError; +use llama_cpp_bindings::mtmd::mtmd_input_part::MtmdInputPart; +use llama_cpp_bindings::mtmd::mtmd_input_text_part::MtmdInputTextPart; +use llama_cpp_bindings::mtmd::mtmd_tokenize_error::MtmdTokenizeError; use llama_cpp_bindings::token::LlamaToken; use llama_cpp_fault_injection::ArmedFault; use llama_cpp_fault_injection::injected_fault_message; diff --git a/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs index 3f8825cc5..0882725b5 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs @@ -1,19 +1,19 @@ use anyhow::Context as _; use anyhow::Result; use anyhow::bail; -use llama_cpp_bindings::BareJsonToolCalls; -use llama_cpp_bindings::ChatAssistantTurn; -use llama_cpp_bindings::ChatSession; -use llama_cpp_bindings::ChatSessionOptions; -use llama_cpp_bindings::ChatToolChoice; -use llama_cpp_bindings::EvalMultimodalChunksParams; +use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::eval_multimodal_chunks_params::EvalMultimodalChunksParams; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::LlamaChatMessage; -use llama_cpp_bindings::mtmd::MtmdBitmap; -use llama_cpp_bindings::mtmd::MtmdInputText; -use llama_cpp_bindings::mtmd::mtmd_default_marker; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_default_marker::mtmd_default_marker; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; diff --git a/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs b/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs index bbc086d48..03a7b4060 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs @@ -1,19 +1,19 @@ use anyhow::Context as _; use anyhow::Result; use anyhow::bail; -use llama_cpp_bindings::EvalMultimodalChunksParams; -use llama_cpp_bindings::TokenUsage; use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_bindings::mtmd::FittedMtmdInputChunk; -use llama_cpp_bindings::mtmd::FittedMtmdInputChunks; -use llama_cpp_bindings::mtmd::MtmdBitmap; -use llama_cpp_bindings::mtmd::MtmdInputChunkType; -use llama_cpp_bindings::mtmd::MtmdInputText; -use llama_cpp_bindings::mtmd::mtmd_default_marker; +use llama_cpp_bindings::eval_multimodal_chunks_params::EvalMultimodalChunksParams; +use llama_cpp_bindings::mtmd::fitted_mtmd_input_chunk::FittedMtmdInputChunk; +use llama_cpp_bindings::mtmd::fitted_mtmd_input_chunks::FittedMtmdInputChunks; +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_default_marker::mtmd_default_marker; +use llama_cpp_bindings::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; use llama_cpp_bindings_tests::prompt_question::PROMPT_QUESTION; +use llama_cpp_bindings_types::TokenUsage; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/parameter_fitting.rs b/llama-cpp-bindings-tests/tests/parameter_fitting.rs index 5d778c9ac..e92070ec6 100644 --- a/llama-cpp-bindings-tests/tests/parameter_fitting.rs +++ b/llama-cpp-bindings-tests/tests/parameter_fitting.rs @@ -4,11 +4,11 @@ use std::pin::pin; use anyhow::Result; use llama_cpp_bindings::context::params::LlamaContextParams; -use llama_cpp_bindings::max_devices; +use llama_cpp_bindings::max_devices::max_devices; use llama_cpp_bindings::model::params::LlamaModelParams; use llama_cpp_bindings::model::params::fit_extra_model::FitExtraModel; -use llama_cpp_bindings::mtmd::MtmdContextParams; -use llama_cpp_bindings::mtmd::MtmdMemoryUsage; +use llama_cpp_bindings::mtmd::mtmd_context_params::MtmdContextParams; +use llama_cpp_bindings::mtmd::mtmd_memory_usage::MtmdMemoryUsage; use llama_cpp_test_harness::GgufSource; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs b/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs index 3f1216a57..a25d238af 100644 --- a/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs +++ b/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs @@ -1,11 +1,11 @@ use anyhow::Context as _; use anyhow::Result; -use llama_cpp_bindings::SampledTokenClassifier; -use llama_cpp_bindings::TokenUsageError; use llama_cpp_bindings::batch_add_error::BatchAddError; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_bindings_types::TokenUsageError; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/reasoning_classification.rs b/llama-cpp-bindings-tests/tests/reasoning_classification.rs index 63f8b01cc..52370bf9d 100644 --- a/llama-cpp-bindings-tests/tests/reasoning_classification.rs +++ b/llama-cpp-bindings-tests/tests/reasoning_classification.rs @@ -1,9 +1,9 @@ use anyhow::Result; -use llama_cpp_bindings::ChatAssistantTurn; -use llama_cpp_bindings::ChatSession; -use llama_cpp_bindings::ChatSessionOptions; -use llama_cpp_bindings::ChatToolChoice; -use llama_cpp_bindings::model::LlamaChatMessage; +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::chat_answer_generation::ChatAnswerGeneration; use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; diff --git a/llama-cpp-bindings-tests/tests/reranking.rs b/llama-cpp-bindings-tests/tests/reranking.rs index 53424dba6..7f15fa8e7 100644 --- a/llama-cpp-bindings-tests/tests/reranking.rs +++ b/llama-cpp-bindings-tests/tests/reranking.rs @@ -1,12 +1,12 @@ use anyhow::Context; use anyhow::Result; use anyhow::bail; -use llama_cpp_bindings::SampledTokenClassifier; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings_tests::normalize::normalize; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs b/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs index 3c68dbdb3..f2ab2e65e 100644 --- a/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs +++ b/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs @@ -1,24 +1,24 @@ use anyhow::Context as _; use anyhow::Result; use anyhow::bail; -use llama_cpp_bindings::BareJsonToolCalls; -use llama_cpp_bindings::GenerationProgress; -use llama_cpp_bindings::IngestError; -use llama_cpp_bindings::ParseChatMessageError; -use llama_cpp_bindings::SampleError; -use llama_cpp_bindings::SampledToken; -use llama_cpp_bindings::SampledTokenClassifier; -use llama_cpp_bindings::TokenOutsideVocabulary; -use llama_cpp_bindings::TokenPiece; -use llama_cpp_bindings::TokenToStringError; +use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::error::ingest_error::IngestError; +use llama_cpp_bindings::error::parse_chat_message_error::ParseChatMessageError; +use llama_cpp_bindings::error::sample_error::SampleError; +use llama_cpp_bindings::error::token_outside_vocabulary::TokenOutsideVocabulary; +use llama_cpp_bindings::error::token_to_string_error::TokenToStringError; +use llama_cpp_bindings::generation_progress::GenerationProgress; use llama_cpp_bindings::ingest_outcome::IngestOutcome; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampled_token_section::SampledTokenSection; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_bindings::token_piece::TokenPiece; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; use llama_cpp_fault_injection::ArmedFault; use llama_cpp_fault_injection::injected_fault_message; diff --git a/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs b/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs index 02e76bdb9..ba646de5c 100644 --- a/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs +++ b/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs @@ -1,8 +1,8 @@ use anyhow::Result; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/session_files.rs b/llama-cpp-bindings-tests/tests/session_files.rs index 5c286b91b..4f5930541 100644 --- a/llama-cpp-bindings-tests/tests/session_files.rs +++ b/llama-cpp-bindings-tests/tests/session_files.rs @@ -1,8 +1,8 @@ use anyhow::Result; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/special_tokens.rs b/llama-cpp-bindings-tests/tests/special_tokens.rs index dc63ee1d7..d6e330ed3 100644 --- a/llama-cpp-bindings-tests/tests/special_tokens.rs +++ b/llama-cpp-bindings-tests/tests/special_tokens.rs @@ -1,8 +1,8 @@ use anyhow::Context; use anyhow::Result; -use llama_cpp_bindings::SampledToken; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/state_snapshots.rs b/llama-cpp-bindings-tests/tests/state_snapshots.rs index e538bfb47..079b4bf35 100644 --- a/llama-cpp-bindings-tests/tests/state_snapshots.rs +++ b/llama-cpp-bindings-tests/tests/state_snapshots.rs @@ -1,8 +1,8 @@ use anyhow::Result; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/token_candidates.rs b/llama-cpp-bindings-tests/tests/token_candidates.rs index 0d4f2f467..1c84cc0d3 100644 --- a/llama-cpp-bindings-tests/tests/token_candidates.rs +++ b/llama-cpp-bindings-tests/tests/token_candidates.rs @@ -2,8 +2,8 @@ use anyhow::Result; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::token::LlamaToken; use llama_cpp_bindings::token::data::LlamaTokenData; use llama_cpp_bindings::token::data_array::LlamaTokenDataArray; diff --git a/llama-cpp-bindings-tests/tests/tokenization.rs b/llama-cpp-bindings-tests/tests/tokenization.rs index 91d5f26c1..5bf4f5d0e 100644 --- a/llama-cpp-bindings-tests/tests/tokenization.rs +++ b/llama-cpp-bindings-tests/tests/tokenization.rs @@ -1,7 +1,7 @@ use anyhow::Result; -use llama_cpp_bindings::SampledToken; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::token::LlamaToken; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/tool_call_classification.rs b/llama-cpp-bindings-tests/tests/tool_call_classification.rs index 678f46c73..1394075fb 100644 --- a/llama-cpp-bindings-tests/tests/tool_call_classification.rs +++ b/llama-cpp-bindings-tests/tests/tool_call_classification.rs @@ -1,9 +1,9 @@ use anyhow::Result; -use llama_cpp_bindings::BareJsonToolCalls; -use llama_cpp_bindings::ChatOutputCompleteness; -use llama_cpp_bindings::SampledToken; -use llama_cpp_bindings::SampledTokenClassifier; +use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; +use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; use llama_cpp_bindings::ingest_outcome::IngestOutcome; +use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings_tests::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; use llama_cpp_bindings_tests::ingested_generation::ingested_generation; use llama_cpp_bindings_tests::recognized_answer::recognized_answer; diff --git a/llama-cpp-bindings-tests/tests/tool_call_grammar.rs b/llama-cpp-bindings-tests/tests/tool_call_grammar.rs index 3d825c874..29ba0d490 100644 --- a/llama-cpp-bindings-tests/tests/tool_call_grammar.rs +++ b/llama-cpp-bindings-tests/tests/tool_call_grammar.rs @@ -1,8 +1,8 @@ use anyhow::Context as _; use anyhow::Result; -use llama_cpp_bindings::ChatOutputCompleteness; -use llama_cpp_bindings::ChatSession; -use llama_cpp_bindings::ChatToolChoice; +use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::chat_answer_generation::ChatAnswerGeneration; use llama_cpp_bindings_tests::paris_weather_tool_call::paris_weather_tool_call; diff --git a/llama-cpp-bindings-tests/tests/tool_call_parsing.rs b/llama-cpp-bindings-tests/tests/tool_call_parsing.rs index b50502c58..0feaaba93 100644 --- a/llama-cpp-bindings-tests/tests/tool_call_parsing.rs +++ b/llama-cpp-bindings-tests/tests/tool_call_parsing.rs @@ -1,17 +1,17 @@ use anyhow::Result; -use llama_cpp_bindings::ChatAssistantTurn; -use llama_cpp_bindings::ChatOutputCompleteness; -use llama_cpp_bindings::ChatSession; -use llama_cpp_bindings::ChatSessionOptions; -use llama_cpp_bindings::ChatToolChoice; -use llama_cpp_bindings::ParsedToolCall; -use llama_cpp_bindings::ParsedToolCallId; -use llama_cpp_bindings::ToolCallArguments; -use llama_cpp_bindings::model::LlamaChatMessage; +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; use llama_cpp_bindings_tests::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; use llama_cpp_bindings_tests::paris_weather_tool_call::paris_weather_tool_call; use llama_cpp_bindings_tests::recognized_answer::recognized_answer; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_bindings_types::ParsedToolCall; +use llama_cpp_bindings_types::ParsedToolCallId; +use llama_cpp_bindings_types::ToolCallArguments; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; use serde_json::json; diff --git a/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs b/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs index 60b60b702..424bf5e15 100644 --- a/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs +++ b/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs @@ -1,12 +1,12 @@ use anyhow::Result; -use llama_cpp_bindings::GenerationProgress; -use llama_cpp_bindings::SampledToken; -use llama_cpp_bindings::SampledTokenClassifier; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::generation_progress::GenerationProgress; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/vocabulary_bounds.rs b/llama-cpp-bindings-tests/tests/vocabulary_bounds.rs index 064abfe2a..49a5dc236 100644 --- a/llama-cpp-bindings-tests/tests/vocabulary_bounds.rs +++ b/llama-cpp-bindings-tests/tests/vocabulary_bounds.rs @@ -1,6 +1,6 @@ -use llama_cpp_bindings::TokenAttrError; -use llama_cpp_bindings::TokenOutsideVocabulary; -use llama_cpp_bindings::TokenToStringError; +use llama_cpp_bindings::error::token_attr_error::TokenAttrError; +use llama_cpp_bindings::error::token_outside_vocabulary::TokenOutsideVocabulary; +use llama_cpp_bindings::error::token_to_string_error::TokenToStringError; use llama_cpp_bindings::token::LlamaToken; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings/src/chat_classification.rs b/llama-cpp-bindings/src/chat_classification.rs index ab951c9b1..b1cd0b028 100644 --- a/llama-cpp-bindings/src/chat_classification.rs +++ b/llama-cpp-bindings/src/chat_classification.rs @@ -29,7 +29,7 @@ impl ChatClassification { #[cfg(test)] mod tests { - use super::ChatClassification; + use crate::chat_classification::ChatClassification; use crate::chat_output_completeness::ChatOutputCompleteness; use crate::chat_piece_role::ChatPieceRole; use crate::sampled_token_section::SampledTokenSection; diff --git a/llama-cpp-bindings/src/chat_grammar_trigger.rs b/llama-cpp-bindings/src/chat_grammar_trigger.rs index a58fbfe2e..abf16862b 100644 --- a/llama-cpp-bindings/src/chat_grammar_trigger.rs +++ b/llama-cpp-bindings/src/chat_grammar_trigger.rs @@ -14,12 +14,12 @@ impl ChatGrammarTrigger { /// /// # Errors /// - /// Returns [`crate::FfiStatusError`] for a trigger type unknown to this binding. + /// Returns [`llama_cpp_ffi_status::FfiStatusError`] for a trigger type unknown to this binding. pub fn from_wrapper( trigger_type: llama_cpp_bindings_sys::llama_rs_chat_grammar_trigger_type, value: String, token: impl FnOnce() -> LlamaToken, - ) -> Result { + ) -> Result { match trigger_type { llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_TOKEN => { Ok(Self::Token(token())) @@ -33,7 +33,7 @@ impl ChatGrammarTrigger { llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL => { Ok(Self::PatternFull(value)) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_chat_session_grammar_trigger_type", code: i64::from(other), }), @@ -43,12 +43,12 @@ impl ChatGrammarTrigger { #[cfg(test)] mod tests { - use super::ChatGrammarTrigger; + use crate::chat_grammar_trigger::ChatGrammarTrigger; use crate::token::LlamaToken; fn trigger_of( trigger_type: llama_cpp_bindings_sys::llama_rs_chat_grammar_trigger_type, - ) -> Result { + ) -> Result { ChatGrammarTrigger::from_wrapper(trigger_type, "".to_owned(), || { LlamaToken::new(151_657) }) @@ -77,7 +77,7 @@ mod tests { fn an_unknown_wrapper_trigger_type_is_a_status_error() { assert_eq!( trigger_of(99), - Err(crate::FfiStatusError { + Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_chat_session_grammar_trigger_type", code: 99, }) diff --git a/llama-cpp-bindings/src/chat_message_parse_outcome.rs b/llama-cpp-bindings/src/chat_message_parse_outcome.rs index 70f0f479c..e4000db4d 100644 --- a/llama-cpp-bindings/src/chat_message_parse_outcome.rs +++ b/llama-cpp-bindings/src/chat_message_parse_outcome.rs @@ -43,7 +43,7 @@ mod tests { use llama_cpp_bindings_types::ToolCallArguments; use serde_json::json; - use super::ChatMessageParseOutcome; + use crate::chat_message_parse_outcome::ChatMessageParseOutcome; use crate::chat_output_completeness::ChatOutputCompleteness; use crate::raw_chat_message::RawChatMessage; diff --git a/llama-cpp-bindings/src/chat_output.rs b/llama-cpp-bindings/src/chat_output.rs index 9a85cabc0..b6e8476a8 100644 --- a/llama-cpp-bindings/src/chat_output.rs +++ b/llama-cpp-bindings/src/chat_output.rs @@ -62,7 +62,7 @@ impl ChatOutput { mod tests { use llama_cpp_bindings_sys::LLAMA_TOKEN_NULL; - use super::ChatOutput; + use crate::chat_output::ChatOutput; use crate::chat_output_token::ChatOutputToken; use crate::token::LlamaToken; diff --git a/llama-cpp-bindings/src/chat_session.rs b/llama-cpp-bindings/src/chat_session.rs index 90fa56902..8083bf544 100644 --- a/llama-cpp-bindings/src/chat_session.rs +++ b/llama-cpp-bindings/src/chat_session.rs @@ -20,20 +20,20 @@ use crate::chat_session_options::ChatSessionOptions; use crate::chat_span::ChatSpan; use crate::chat_span_kind::ChatSpanKind; use crate::chat_spans::ChatSpans; -use crate::chat_template_tool_calls::known_marker_candidates; +use crate::chat_template_tool_calls::known_marker_candidates::known_marker_candidates; use crate::cpp_text_view::cpp_text_view; use crate::error::chat_session_error::ChatSessionError; use crate::error::parse_chat_message_error::ParseChatMessageError; use crate::lazy_grammar_triggers::LazyGrammarTriggers; -use crate::model::AddBos; use crate::model::LlamaModel; -use crate::model::ParseSpecialTokens; +use crate::model::add_bos::AddBos; use crate::model::llama_chat_message::LlamaChatMessage; +use crate::model::parse_special_tokens::ParseSpecialTokens; use crate::parsed_chat::take_parsed_chat_message; use crate::raw_chat_message::RawChatMessage; use crate::sampling::LlamaSampler; use crate::token::LlamaToken; -use crate::tool_call_format::ToolCallFormatOutcome; +use crate::tool_call_format::tool_call_format_outcome::ToolCallFormatOutcome; use crate::tool_call_format::try_parse; const GRAMMAR_ROOT: &str = "root"; @@ -87,8 +87,8 @@ fn session_text(text: *const c_char, length: usize) -> Result crate::FfiContractError { - crate::FfiContractError { operation, detail } +) -> llama_cpp_ffi_status::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation, detail } } fn chat_session_create_status_to_result( @@ -167,7 +167,7 @@ fn chat_session_create_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_ERROR_ARG => { Err(contract_violation(OPERATION, "was given a null out_error argument").into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: OPERATION, code: i64::from(other), } @@ -231,7 +231,7 @@ fn chat_session_parse_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_ERROR_ARG => { Err(contract_violation(OPERATION, "was given a null out_error argument").into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: OPERATION, code: i64::from(other), } @@ -256,7 +256,7 @@ fn read_chat_spans( }) .map(|kind| ChatSpan { kind, bytes }) }) - .collect::, crate::FfiStatusError>>() + .collect::, llama_cpp_ffi_status::FfiStatusError>>() .map(|spans_read| ChatSpans { parsed_length: unsafe { llama_cpp_bindings_sys::llama_rs_chat_spans_parsed_length(spans.as_ptr()) @@ -321,7 +321,7 @@ fn chat_session_classify_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_ERROR_ARG => { Err(contract_violation(OPERATION, "was given a null out_error argument").into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: OPERATION, code: i64::from(other), } @@ -435,7 +435,7 @@ impl<'model> ChatSession<'model> { .collect() } - /// The rendered prompt as one string, for [`crate::mtmd::MtmdContext::tokenize`], which + /// The rendered prompt as one string, for [`crate::mtmd::mtmd_context::MtmdContext::tokenize`], which /// finds the media markers in it the way llama-server hands a multimodal prompt to mtmd. /// /// # Errors @@ -824,10 +824,12 @@ mod tests { assert_eq!( chat_session_create_status_to_result(status, ptr::null_mut(), ptr::null_mut()) .err(), - Some(ChatSessionError::FfiContract(crate::FfiContractError { - operation: "llama_rs_chat_session_create", - detail - })) + Some(ChatSessionError::FfiContract( + llama_cpp_ffi_status::FfiContractError { + operation: "llama_rs_chat_session_create", + detail + } + )) ); } } @@ -888,10 +890,12 @@ mod tests { chat_session_create_status_to_result(status, ptr::null_mut(), ptr::null_mut()).err() }), refusals.map(|_status| { - Some(ChatSessionError::FfiContract(crate::FfiContractError { - operation: "llama_rs_chat_session_create", - detail: "reported a failure without an error message", - })) + Some(ChatSessionError::FfiContract( + llama_cpp_ffi_status::FfiContractError { + operation: "llama_rs_chat_session_create", + detail: "reported a failure without an error message", + }, + )) }) ); } @@ -920,10 +924,12 @@ mod tests { fn creating_a_session_preserves_an_unknown_status() { assert_eq!( chat_session_create_status_to_result(255, ptr::null_mut(), ptr::null_mut()).err(), - Some(ChatSessionError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_chat_session_create", - code: 255 - })) + Some(ChatSessionError::FfiStatus( + llama_cpp_ffi_status::FfiStatusError { + operation: "llama_rs_chat_session_create", + code: 255 + } + )) ); } @@ -964,7 +970,7 @@ mod tests { }; let contract = |detail| { Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_chat_session_parse", detail, }, @@ -1023,7 +1029,7 @@ mod tests { contract("was given a null byte_tokens argument"), contract("was given a null out_handle argument"), contract("was given a null out_error argument"), - Some(ParseChatMessageError::FfiStatus(crate::FfiStatusError { operation: "llama_rs_chat_session_parse", code: 255 })), + Some(ParseChatMessageError::FfiStatus(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_chat_session_parse", code: 255 })), ] ); } @@ -1056,7 +1062,7 @@ mod tests { }; let contract = |detail| { Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_chat_session_classify", detail, }, @@ -1115,7 +1121,7 @@ mod tests { contract("was given a null byte_tokens argument"), contract("was given a null out_spans argument"), contract("was given a null out_error argument"), - Some(ParseChatMessageError::FfiStatus(crate::FfiStatusError { operation: "llama_rs_chat_session_classify", code: 255 })), + Some(ParseChatMessageError::FfiStatus(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_chat_session_classify", code: 255 })), ] ); } @@ -1210,15 +1216,15 @@ mod tests { .err(), ], [ - Some(ParseChatMessageError::FfiContract(crate::FfiContractError { + Some(ParseChatMessageError::FfiContract(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_chat_session_parse", detail: "reported a failure without an error message", })), - Some(ParseChatMessageError::FfiContract(crate::FfiContractError { + Some(ParseChatMessageError::FfiContract(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_chat_session_parse", detail: "reported a failure without an error message", })), - Some(ParseChatMessageError::FfiContract(crate::FfiContractError { + Some(ParseChatMessageError::FfiContract(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_chat_session_classify", detail: "reported a thrown C++ exception without an error message", })), diff --git a/llama-cpp-bindings/src/chat_span_kind.rs b/llama-cpp-bindings/src/chat_span_kind.rs index 952375a8e..86f8b264f 100644 --- a/llama-cpp-bindings/src/chat_span_kind.rs +++ b/llama-cpp-bindings/src/chat_span_kind.rs @@ -24,7 +24,7 @@ impl ChatSpanKind { } impl TryFrom for ChatSpanKind { - type Error = crate::FfiStatusError; + type Error = llama_cpp_ffi_status::FfiStatusError; fn try_from( kind: llama_cpp_bindings_sys::llama_rs_chat_span_kind, @@ -39,7 +39,7 @@ impl TryFrom for ChatSpanKind { llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL_MARKER => { Ok(Self::ToolCallMarker) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_chat_spans_kind", code: i64::from(other), }), @@ -107,7 +107,7 @@ mod tests { fn an_unknown_wrapper_span_kind_is_a_status_error() { assert_eq!( ChatSpanKind::try_from(99), - Err(crate::FfiStatusError { + Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_chat_spans_kind", code: 99, }) diff --git a/llama-cpp-bindings/src/chat_template_tool_calls.rs b/llama-cpp-bindings/src/chat_template_tool_calls.rs index 4cad316e3..cd9732918 100644 --- a/llama-cpp-bindings/src/chat_template_tool_calls.rs +++ b/llama-cpp-bindings/src/chat_template_tool_calls.rs @@ -4,5 +4,3 @@ pub mod known_marker_candidates; pub mod mistral3_arrow_args; pub mod qwen3_json_inside_tool_call; pub mod qwen_xml_tags; - -pub use known_marker_candidates::known_marker_candidates; diff --git a/llama-cpp-bindings/src/chat_templates.rs b/llama-cpp-bindings/src/chat_templates.rs index 981d74493..156100324 100644 --- a/llama-cpp-bindings/src/chat_templates.rs +++ b/llama-cpp-bindings/src/chat_templates.rs @@ -15,7 +15,7 @@ fn chat_templates_init_status_to_result( match status { llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_OK => NonNull::new(out_templates) .ok_or_else(|| { - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_chat_templates_init", detail: "success status contained a null templates handle", } @@ -41,27 +41,27 @@ fn chat_templates_init_status_to_result( Err(ChatTemplatesError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_MODEL_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_chat_templates_init", detail: "was given a null model argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_TEMPLATES_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_chat_templates_init", detail: "was given a null out_templates argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_chat_templates_init", detail: "was given a null out_error argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_chat_templates_init", code: i64::from(other), } @@ -139,7 +139,7 @@ mod tests { } const fn contract(detail: &'static str) -> ChatTemplatesError { - ChatTemplatesError::FfiContract(crate::FfiContractError { + ChatTemplatesError::FfiContract(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_chat_templates_init", detail, }) @@ -194,7 +194,7 @@ mod tests { Some(contract("was given a null model argument")), Some(contract("was given a null out_templates argument")), Some(contract("was given a null out_error argument")), - Some(ChatTemplatesError::FfiStatus(crate::FfiStatusError { + Some(ChatTemplatesError::FfiStatus(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_chat_templates_init", code: 255, })), diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index 58d5c61d5..cb787621b 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -14,10 +14,16 @@ use llama_cpp_ffi_status::read_and_free_cpp_string; use crate::context::context_limits::ContextLimits; use crate::context::decoded_output_rows::DecodedOutputRows; use crate::context::decoded_outputs::DecodedOutputs; +use crate::context::llama_pooling_type::LlamaPoolingType; use crate::context::params::LlamaContextParams; -use crate::context::params::LlamaPoolingType; use crate::context::sequence_output_capacity::SequenceOutputCapacity; use crate::error::batch_transfer_error::BatchTransferError; +use crate::error::decode_error::DecodeError; +use crate::error::embeddings_error::EmbeddingsError; +use crate::error::encode_error::EncodeError; +use crate::error::llama_context_load_error::LlamaContextLoadError; +use crate::error::llama_lora_adapters_error::LlamaLoraAdaptersError; +use crate::error::logits_error::LogitsError; use crate::error::process_failure::ProcessFailure; use crate::llama_backend::LlamaBackend; use crate::llama_batch::LlamaBatch; @@ -26,10 +32,6 @@ use crate::model::lora_adapter_scale::LoraAdapterScale; use crate::token::LlamaToken; use crate::token::data::LlamaTokenData; use crate::token::data_array::LlamaTokenDataArray; -use crate::{ - DecodeError, EmbeddingsError, EncodeError, LlamaContextLoadError, LlamaLoraAdaptersError, - LogitsError, -}; const fn check_lora_adapters_result(err_code: i32) -> Result<(), LlamaLoraAdaptersError> { if err_code != 0 { @@ -49,21 +51,21 @@ fn nextn_hidden_state_probe_status_to_result( Ok(produces_nextn_hidden_state) } llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_CTX_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_context_graph_produces_nextn_hidden_state", detail: "was given a null ctx argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_PRODUCES_NEXTN_HIDDEN_STATE_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_context_graph_produces_nextn_hidden_state", detail: "was given a null out_produces_nextn_hidden_state argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_context_graph_produces_nextn_hidden_state", detail: "was given a null out_error argument", } @@ -88,7 +90,7 @@ fn nextn_hidden_state_probe_status_to_result( }?; Err(EmbeddingsError::NextnProbeReported { message }) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_context_graph_produces_nextn_hidden_state", code: i64::from(other), } @@ -104,7 +106,7 @@ fn new_context_with_model_status_to_result( match status { llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_OK => NonNull::new(out_ctx) .ok_or_else(|| { - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_new_context_with_model", detail: "success status contained a null context", } @@ -130,27 +132,27 @@ fn new_context_with_model_status_to_result( Err(LlamaContextLoadError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_NULL_MODEL_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_new_context_with_model", detail: "was given a null model argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_NULL_OUT_CTX_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_new_context_with_model", detail: "was given a null out_ctx argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_new_context_with_model", detail: "was given a null out_error argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_new_context_with_model", code: i64::from(other), } @@ -166,7 +168,7 @@ fn batch_ext_init_status_to_result( match status { llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_OK => { NonNull::new(out_batch).ok_or_else(|| { - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_batch_ext_init", detail: "success status contained a null batch", } @@ -190,27 +192,27 @@ fn batch_ext_init_status_to_result( Err(LlamaContextLoadError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_NULL_CTX_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_batch_ext_init", detail: "was given a null ctx argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_NULL_OUT_BATCH_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_batch_ext_init", detail: "was given a null out_batch argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_batch_ext_init", detail: "was given a null out_error argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_batch_ext_init", code: i64::from(other), } @@ -223,7 +225,7 @@ fn batch_ext_add_entry_status_to_result( out_error: *mut c_char, ) -> Result<(), BatchTransferError> { let refused_validated_entry = |detail: &'static str| -> Result<(), BatchTransferError> { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_batch_ext_add_entry", detail, } @@ -281,7 +283,7 @@ fn batch_ext_add_entry_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_OUT_ERROR_ARG => { refused_validated_entry("was given a null out_error argument") } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_batch_ext_add_entry", code: i64::from(other), } @@ -300,7 +302,7 @@ where const OPERATION: &str = "llama_rs_process"; let contract_violation = |detail: &'static str| { - ProcessFailure::from(crate::FfiContractError { + ProcessFailure::from(llama_cpp_ffi_status::FfiContractError { operation: OPERATION, detail, }) @@ -345,7 +347,7 @@ where llama_cpp_bindings_sys::LLAMA_RS_PROCESS_NULL_OUT_ERROR_ARG => { Err(contract_violation("was given a null out_error argument").into()) } - other => Err(ProcessFailure::from(crate::FfiStatusError { + other => Err(ProcessFailure::from(llama_cpp_ffi_status::FfiStatusError { operation: OPERATION, code: i64::from(other), }) @@ -955,7 +957,7 @@ impl Drop for LlamaContext<'_> { #[cfg(test)] mod check_lora_adapters_result_tests { use super::check_lora_adapters_result; - use crate::LlamaLoraAdaptersError; + use crate::error::llama_lora_adapters_error::LlamaLoraAdaptersError; #[test] fn check_lora_adapters_result_ok_for_zero() { @@ -978,7 +980,7 @@ mod nextn_hidden_state_probe_status_to_result_tests { use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer; use super::nextn_hidden_state_probe_status_to_result; - use crate::EmbeddingsError; + use crate::error::embeddings_error::EmbeddingsError; #[test] fn nextn_hidden_state_probe_status_to_result_maps_ok_to_whether_the_graph_produces_it() { @@ -1000,7 +1002,7 @@ mod nextn_hidden_state_probe_status_to_result_tests { false, ptr::null_mut(), ), - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_context_graph_produces_nextn_hidden_state", detail: "was given a null ctx argument", } @@ -1016,7 +1018,7 @@ mod nextn_hidden_state_probe_status_to_result_tests { false, ptr::null_mut(), ), - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_context_graph_produces_nextn_hidden_state", detail: "was given a null out_produces_nextn_hidden_state argument", } @@ -1032,7 +1034,7 @@ mod nextn_hidden_state_probe_status_to_result_tests { false, ptr::null_mut(), ), - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_context_graph_produces_nextn_hidden_state", detail: "was given a null out_error argument", } @@ -1108,7 +1110,7 @@ mod nextn_hidden_state_probe_status_to_result_tests { false, ptr::null_mut(), ), - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_context_graph_produces_nextn_hidden_state", detail: "reported a thrown C++ exception without an error message", } @@ -1120,7 +1122,7 @@ mod nextn_hidden_state_probe_status_to_result_tests { fn nextn_hidden_state_probe_status_to_result_maps_an_unknown_status() { assert_eq!( nextn_hidden_state_probe_status_to_result(255, false, ptr::null_mut()), - Err(crate::FfiStatusError { + Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_context_graph_produces_nextn_hidden_state", code: 255, } @@ -1134,7 +1136,7 @@ mod new_context_with_model_status_to_result_tests { use std::ptr; use super::new_context_with_model_status_to_result; - use crate::LlamaContextLoadError; + use crate::error::llama_context_load_error::LlamaContextLoadError; #[test] fn new_context_success_with_null_context_is_contract_error() { @@ -1147,7 +1149,7 @@ mod new_context_with_model_status_to_result_tests { assert_eq!( result, Err(LlamaContextLoadError::FfiContract( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_new_context_with_model", detail: "success status contained a null context", } @@ -1206,7 +1208,7 @@ mod new_context_with_model_status_to_result_tests { assert_eq!( result, - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_new_context_with_model", detail: "reported a thrown C++ exception without an error message", } @@ -1220,10 +1222,12 @@ mod new_context_with_model_status_to_result_tests { assert_eq!( result, - Err(LlamaContextLoadError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_new_context_with_model", - code: 255, - })) + Err(LlamaContextLoadError::FfiStatus( + llama_cpp_ffi_status::FfiStatusError { + operation: "llama_rs_new_context_with_model", + code: 255, + } + )) ); } @@ -1237,7 +1241,7 @@ mod new_context_with_model_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_new_context_with_model", detail: "was given a null model argument", } @@ -1256,7 +1260,7 @@ mod new_context_with_model_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_new_context_with_model", detail: "was given a null out_ctx argument", } @@ -1275,7 +1279,7 @@ mod new_context_with_model_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_new_context_with_model", detail: "was given a null out_error argument", } @@ -1303,7 +1307,7 @@ mod batch_ext_init_status_to_result_tests { use std::ptr; use super::batch_ext_init_status_to_result; - use crate::LlamaContextLoadError; + use crate::error::llama_context_load_error::LlamaContextLoadError; #[test] fn initializing_a_batch_reports_each_null_pointer_as_a_contract_violation() { @@ -1328,7 +1332,7 @@ mod batch_ext_init_status_to_result_tests { assert_eq!( batch_ext_init_status_to_result(status, ptr::null_mut(), ptr::null_mut()).err(), Some(LlamaContextLoadError::FfiContract( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_batch_ext_init", detail, } @@ -1386,7 +1390,7 @@ mod batch_ext_init_status_to_result_tests { ) .err(), Some(LlamaContextLoadError::FfiContract( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_batch_ext_init", detail: "reported a thrown C++ exception without an error message", } @@ -1398,10 +1402,12 @@ mod batch_ext_init_status_to_result_tests { fn initializing_a_batch_preserves_an_unknown_status() { assert_eq!( batch_ext_init_status_to_result(255, ptr::null_mut(), ptr::null_mut()).err(), - Some(LlamaContextLoadError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_batch_ext_init", - code: 255, - })) + Some(LlamaContextLoadError::FfiStatus( + llama_cpp_ffi_status::FfiStatusError { + operation: "llama_rs_batch_ext_init", + code: 255, + } + )) ); } } @@ -1463,7 +1469,7 @@ mod batch_ext_add_entry_status_to_result_tests { ] { assert_eq!( batch_ext_add_entry_status_to_result(status, ptr::null_mut()), - Err(BatchTransferError::FfiContract(crate::FfiContractError { + Err(BatchTransferError::FfiContract(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_batch_ext_add_entry", detail, })) @@ -1510,10 +1516,12 @@ mod batch_ext_add_entry_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), ), - Err(BatchTransferError::FfiContract(crate::FfiContractError { - operation: "llama_rs_batch_ext_add_entry", - detail: "reported a thrown C++ exception without an error message", - })) + Err(BatchTransferError::FfiContract( + llama_cpp_ffi_status::FfiContractError { + operation: "llama_rs_batch_ext_add_entry", + detail: "reported a thrown C++ exception without an error message", + } + )) ); } @@ -1521,10 +1529,12 @@ mod batch_ext_add_entry_status_to_result_tests { fn adding_a_batch_entry_preserves_an_unknown_status() { assert_eq!( batch_ext_add_entry_status_to_result(255, ptr::null_mut()), - Err(BatchTransferError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_batch_ext_add_entry", - code: 255, - })) + Err(BatchTransferError::FfiStatus( + llama_cpp_ffi_status::FfiStatusError { + operation: "llama_rs_batch_ext_add_entry", + code: 255, + } + )) ); } } @@ -1534,14 +1544,14 @@ mod process_status_to_result_tests { use std::ptr; use super::process_status_to_result; - use crate::DecodeError; - use crate::EncodeError; + use crate::error::decode_error::DecodeError; + use crate::error::encode_error::EncodeError; use crate::error::process_failure::ProcessFailure; const OPERATION: &str = "llama_rs_process"; fn contract_violation(detail: &'static str) -> DecodeError { - ProcessFailure::from(crate::FfiContractError { + ProcessFailure::from(llama_cpp_ffi_status::FfiContractError { operation: OPERATION, detail, }) @@ -1585,7 +1595,7 @@ mod process_status_to_result_tests { "was given a null out_llama_cpp_return_code argument" )), Err(contract_violation("was given a null out_error argument")), - Err(ProcessFailure::from(crate::FfiStatusError { + Err(ProcessFailure::from(llama_cpp_ffi_status::FfiStatusError { operation: OPERATION, code: 255, }) @@ -1645,7 +1655,7 @@ mod logits_slice_tests { use std::ptr::NonNull; use super::logits_slice; - use crate::LogitsError; + use crate::error::logits_error::LogitsError; #[test] fn logits_slice_from_negative_vocab_maps_vocab_size_overflow() { diff --git a/llama-cpp-bindings/src/context/decoded_outputs.rs b/llama-cpp-bindings/src/context/decoded_outputs.rs index 376f2c77d..b2b75d3ca 100644 --- a/llama-cpp-bindings/src/context/decoded_outputs.rs +++ b/llama-cpp-bindings/src/context/decoded_outputs.rs @@ -1,6 +1,6 @@ -use crate::LogitsError; use crate::context::decoded_output_rows::DecodedOutputRows; use crate::context::llama_pooling_type::LlamaPoolingType; +use crate::error::logits_error::LogitsError; #[derive(Clone, Debug, Eq, PartialEq)] pub struct DecodedOutputs { @@ -53,9 +53,9 @@ impl DecodedOutputs { #[cfg(test)] mod tests { use super::DecodedOutputs; - use crate::LogitsError; use crate::context::decoded_output_rows::DecodedOutputRows; use crate::context::llama_pooling_type::LlamaPoolingType; + use crate::error::logits_error::LogitsError; fn outputs_at(output_token_indices: Vec) -> DecodedOutputs { DecodedOutputs { diff --git a/llama-cpp-bindings/src/context/memory.rs b/llama-cpp-bindings/src/context/memory.rs index a3c8af08a..4811a085d 100644 --- a/llama-cpp-bindings/src/context/memory.rs +++ b/llama-cpp-bindings/src/context/memory.rs @@ -4,9 +4,11 @@ use std::os::raw::c_char; use std::ptr; use crate::context::LlamaContext; -use crate::error::{ - MemorySeqAddError, MemorySeqCpError, MemorySeqDivError, MemorySeqPosMaxError, MemorySeqRmError, -}; +use crate::error::memory_seq_add_error::MemorySeqAddError; +use crate::error::memory_seq_cp_error::MemorySeqCpError; +use crate::error::memory_seq_div_error::MemorySeqDivError; +use crate::error::memory_seq_pos_max_error::MemorySeqPosMaxError; +use crate::error::memory_seq_rm_error::MemorySeqRmError; use llama_cpp_ffi_status::read_and_free_cpp_string; fn memory_seq_add_status_to_result( @@ -38,20 +40,20 @@ fn memory_seq_add_status_to_result( Err(MemorySeqAddError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_CTX_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_memory_seq_add", detail: "was given a null ctx argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_MODEL => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_memory_seq_add", detail: "was given a null model argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_memory_seq_add", code: i64::from(other), } @@ -88,20 +90,20 @@ fn memory_seq_div_status_to_result( Err(MemorySeqDivError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_CTX_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_memory_seq_div", detail: "was given a null ctx argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_MODEL => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_memory_seq_div", detail: "was given a null model argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_memory_seq_div", code: i64::from(other), } @@ -118,21 +120,21 @@ fn memory_seq_pos_max_status_to_result( match status { llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_OK => Ok(position), llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_CTX_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_memory_seq_pos_max", detail: "context pointer was null", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_OUT_POSITION_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_memory_seq_pos_max", detail: "output position pointer was null", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_memory_seq_pos_max", detail: "output error pointer was null", } @@ -160,7 +162,7 @@ fn memory_seq_pos_max_status_to_result( }?; Err(MemorySeqPosMaxError::Reported { message }) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_memory_seq_pos_max", code: i64::from(other), } @@ -336,7 +338,7 @@ mod memory_seq_add_status_to_result_tests { use std::ptr; use super::memory_seq_add_status_to_result; - use crate::error::MemorySeqAddError; + use crate::error::memory_seq_add_error::MemorySeqAddError; #[test] fn add_ok_status_maps_to_ok() { @@ -388,7 +390,7 @@ mod memory_seq_add_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), ), - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_memory_seq_add", detail: "reported a thrown C++ exception without an error message", } @@ -418,10 +420,12 @@ mod memory_seq_add_status_to_result_tests { assert_eq!( result, - Err(MemorySeqAddError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_memory_seq_add", - code: 255, - })) + Err(MemorySeqAddError::FfiStatus( + llama_cpp_ffi_status::FfiStatusError { + operation: "llama_rs_memory_seq_add", + code: 255, + } + )) ); } @@ -434,7 +438,7 @@ mod memory_seq_add_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_memory_seq_add", detail: "was given a null ctx argument", } @@ -452,7 +456,7 @@ mod memory_seq_add_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_memory_seq_add", detail: "was given a null model argument", } @@ -476,7 +480,7 @@ mod memory_seq_div_status_to_result_tests { use std::ptr; use super::memory_seq_div_status_to_result; - use crate::error::MemorySeqDivError; + use crate::error::memory_seq_div_error::MemorySeqDivError; #[test] fn div_ok_status_maps_to_ok() { @@ -528,7 +532,7 @@ mod memory_seq_div_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), ), - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_memory_seq_div", detail: "reported a thrown C++ exception without an error message", } @@ -558,10 +562,12 @@ mod memory_seq_div_status_to_result_tests { assert_eq!( result, - Err(MemorySeqDivError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_memory_seq_div", - code: 255, - })) + Err(MemorySeqDivError::FfiStatus( + llama_cpp_ffi_status::FfiStatusError { + operation: "llama_rs_memory_seq_div", + code: 255, + } + )) ); } @@ -574,7 +580,7 @@ mod memory_seq_div_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_memory_seq_div", detail: "was given a null ctx argument", } @@ -592,7 +598,7 @@ mod memory_seq_div_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_memory_seq_div", detail: "was given a null model argument", } @@ -616,7 +622,7 @@ mod memory_seq_pos_max_status_to_result_tests { use std::ptr; use super::memory_seq_pos_max_status_to_result; - use crate::error::MemorySeqPosMaxError; + use crate::error::memory_seq_pos_max_error::MemorySeqPosMaxError; #[test] fn seq_pos_max_ok_status_returns_position() { @@ -640,10 +646,12 @@ mod memory_seq_pos_max_status_to_result_tests { 2, ptr::null_mut(), ), - Err(MemorySeqPosMaxError::FfiContract(crate::FfiContractError { - operation: "llama_rs_memory_seq_pos_max", - detail: "context pointer was null", - })) + Err(MemorySeqPosMaxError::FfiContract( + llama_cpp_ffi_status::FfiContractError { + operation: "llama_rs_memory_seq_pos_max", + detail: "context pointer was null", + } + )) ); } @@ -656,10 +664,12 @@ mod memory_seq_pos_max_status_to_result_tests { 2, ptr::null_mut(), ), - Err(MemorySeqPosMaxError::FfiContract(crate::FfiContractError { - operation: "llama_rs_memory_seq_pos_max", - detail: "output position pointer was null", - })) + Err(MemorySeqPosMaxError::FfiContract( + llama_cpp_ffi_status::FfiContractError { + operation: "llama_rs_memory_seq_pos_max", + detail: "output position pointer was null", + } + )) ); } @@ -672,10 +682,12 @@ mod memory_seq_pos_max_status_to_result_tests { 2, ptr::null_mut(), ), - Err(MemorySeqPosMaxError::FfiContract(crate::FfiContractError { - operation: "llama_rs_memory_seq_pos_max", - detail: "output error pointer was null", - })) + Err(MemorySeqPosMaxError::FfiContract( + llama_cpp_ffi_status::FfiContractError { + operation: "llama_rs_memory_seq_pos_max", + detail: "output error pointer was null", + } + )) ); } @@ -727,7 +739,7 @@ mod memory_seq_pos_max_status_to_result_tests { 2, ptr::null_mut(), ), - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_memory_seq_pos_max", detail: "reported a thrown C++ exception without an error message", } @@ -758,10 +770,12 @@ mod memory_seq_pos_max_status_to_result_tests { fn seq_pos_max_unknown_status_is_preserved() { assert_eq!( memory_seq_pos_max_status_to_result(255, -1, 2, ptr::null_mut(),), - Err(MemorySeqPosMaxError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_memory_seq_pos_max", - code: 255, - })) + Err(MemorySeqPosMaxError::FfiStatus( + llama_cpp_ffi_status::FfiStatusError { + operation: "llama_rs_memory_seq_pos_max", + code: 255, + } + )) ); } diff --git a/llama-cpp-bindings/src/context/params.rs b/llama-cpp-bindings/src/context/params.rs index 4ad588309..d2636fafe 100644 --- a/llama-cpp-bindings/src/context/params.rs +++ b/llama-cpp-bindings/src/context/params.rs @@ -1,11 +1,10 @@ +use crate::context::kv_cache_type::KvCacheType; +use crate::context::llama_attention_type::LlamaAttentionType; +use crate::context::llama_pooling_type::LlamaPoolingType; +use crate::context::rope_scaling_type::RopeScalingType; use std::fmt::Debug; use std::num::NonZeroU32; -pub use crate::context::kv_cache_type::KvCacheType; -pub use crate::context::llama_attention_type::LlamaAttentionType; -pub use crate::context::llama_pooling_type::LlamaPoolingType; -pub use crate::context::rope_scaling_type::RopeScalingType; - #[derive(Debug, Clone, Copy)] pub struct LlamaContextParams { pub context_params: llama_cpp_bindings_sys::llama_context_params, @@ -333,7 +332,10 @@ impl Default for LlamaContextParams { #[cfg(test)] mod tests { - use super::{KvCacheType, LlamaAttentionType, LlamaPoolingType, RopeScalingType}; + use super::KvCacheType; + use super::LlamaAttentionType; + use super::LlamaPoolingType; + use super::RopeScalingType; #[test] fn default_params_have_expected_values() { diff --git a/llama-cpp-bindings/src/context/sequence_output_capacity.rs b/llama-cpp-bindings/src/context/sequence_output_capacity.rs index 5192bf13d..592ae1a8d 100644 --- a/llama-cpp-bindings/src/context/sequence_output_capacity.rs +++ b/llama-cpp-bindings/src/context/sequence_output_capacity.rs @@ -1,5 +1,5 @@ -use crate::LlamaContextLoadError; -use crate::context::params::LlamaAttentionType; +use crate::context::llama_attention_type::LlamaAttentionType; +use crate::error::llama_context_load_error::LlamaContextLoadError; pub struct SequenceOutputCapacity { pub attention_type: LlamaAttentionType, @@ -53,8 +53,8 @@ impl SequenceOutputCapacity { #[cfg(test)] mod tests { use super::SequenceOutputCapacity; - use crate::LlamaContextLoadError; - use crate::context::params::LlamaAttentionType; + use crate::context::llama_attention_type::LlamaAttentionType; + use crate::error::llama_context_load_error::LlamaContextLoadError; const TWO_SEQUENCES_PAST_ONE_OUTPUT: Result<(), LlamaContextLoadError> = Err(LlamaContextLoadError::SequencesExceedOutputCapacity { diff --git a/llama-cpp-bindings/src/context/session.rs b/llama-cpp-bindings/src/context/session.rs index 25bd6a2b3..09ad084e2 100644 --- a/llama-cpp-bindings/src/context/session.rs +++ b/llama-cpp-bindings/src/context/session.rs @@ -21,27 +21,29 @@ fn state_data_status_to_result( ) -> Result { match status { llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_OK => Ok(byte_count), - llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_CTX_ARG => Err(crate::FfiContractError { - operation, - detail: "was given a null ctx argument", + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_CTX_ARG => { + Err(llama_cpp_ffi_status::FfiContractError { + operation, + detail: "was given a null ctx argument", + } + .into()) } - .into()), llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_BUFFER_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation, detail: "was given a null buffer argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_OUT_BYTE_COUNT_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation, detail: "was given a null out_byte_count argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation, detail: "was given a null out_error argument", } @@ -64,7 +66,7 @@ fn state_data_status_to_result( Err(StateDataError::Reported { message }) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation, code: i64::from(other), } @@ -520,10 +522,12 @@ mod state_data_status_to_result_tests { ] { assert_eq!( state_data_status_to_result(status, 0, ptr::null_mut(), "llama_rs_state_get_data"), - Err(StateDataError::FfiContract(crate::FfiContractError { - operation: "llama_rs_state_get_data", - detail, - })) + Err(StateDataError::FfiContract( + llama_cpp_ffi_status::FfiContractError { + operation: "llama_rs_state_get_data", + detail, + } + )) ); } } @@ -568,10 +572,12 @@ mod state_data_status_to_result_tests { ptr::null_mut(), "llama_rs_state_get_data", ), - Err(StateDataError::FfiContract(crate::FfiContractError { - operation: "llama_rs_state_get_data", - detail: "reported a thrown C++ exception without an error message", - })) + Err(StateDataError::FfiContract( + llama_cpp_ffi_status::FfiContractError { + operation: "llama_rs_state_get_data", + detail: "reported a thrown C++ exception without an error message", + } + )) ); } @@ -579,10 +585,12 @@ mod state_data_status_to_result_tests { fn state_data_status_to_result_preserves_an_unknown_status() { assert_eq!( state_data_status_to_result(255, 0, ptr::null_mut(), "llama_rs_state_get_data"), - Err(StateDataError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_state_get_data", - code: 255, - })) + Err(StateDataError::FfiStatus( + llama_cpp_ffi_status::FfiStatusError { + operation: "llama_rs_state_get_data", + code: 255, + } + )) ); } } diff --git a/llama-cpp-bindings/src/context/state_data_error.rs b/llama-cpp-bindings/src/context/state_data_error.rs index 0487b2980..f80ba3b4a 100644 --- a/llama-cpp-bindings/src/context/state_data_error.rs +++ b/llama-cpp-bindings/src/context/state_data_error.rs @@ -1,9 +1,9 @@ #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum StateDataError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/dry_sequence_breakers.rs b/llama-cpp-bindings/src/dry_sequence_breakers.rs index 640ca0403..7ce4ba658 100644 --- a/llama-cpp-bindings/src/dry_sequence_breakers.rs +++ b/llama-cpp-bindings/src/dry_sequence_breakers.rs @@ -3,7 +3,7 @@ use std::ffi::CString; use llama_cpp_bindings_sys::LLAMA_DRY_MAX_SEQ_BREAKERS; -use crate::SamplingError; +use crate::error::sampling_error::SamplingError; const DRY_SEQUENCE_BREAKER_MAX_BYTES: usize = 40; @@ -58,7 +58,7 @@ mod tests { use std::ffi::CString; use super::dry_sequence_breakers; - use crate::SamplingError; + use crate::error::sampling_error::SamplingError; fn distinct_breakers(count: usize) -> Vec { (0..count) diff --git a/llama-cpp-bindings/src/error.rs b/llama-cpp-bindings/src/error.rs index 5f8caf431..0dc0c7d6e 100644 --- a/llama-cpp-bindings/src/error.rs +++ b/llama-cpp-bindings/src/error.rs @@ -43,50 +43,6 @@ pub mod token_to_string_error; pub mod tool_call_format_failure; pub mod xml_function_tags_failure; -pub use llama_cpp_ffi_status::FfiContractError; -pub use llama_cpp_ffi_status::FfiStatusError; - -pub use bracketed_args_failure::BracketedArgsFailure; -pub use chat_session_error::ChatSessionError; -pub use chat_template_error::ChatTemplateError; -pub use chat_templates_error::ChatTemplatesError; -pub use decode_error::DecodeError; -pub use embeddings_error::EmbeddingsError; -pub use encode_error::EncodeError; -pub use eval_multimodal_chunks_error::EvalMultimodalChunksError; -pub use fit_error::FitError; -pub use grammar_error::GrammarError; -pub use grammar_runtime_error::GrammarRuntimeError; -pub use ingest_error::IngestError; -pub use json_object_failure::JsonObjectFailure; -pub use json_schema_to_grammar_error::JsonSchemaToGrammarError; -pub use key_value_xml_tags_failure::KeyValueXmlTagsFailure; -pub use llama_context_load_error::LlamaContextLoadError; -pub use llama_cpp_error::LlamaCppError; -pub use llama_lora_adapter_init_error::LlamaLoraAdapterInitError; -pub use llama_lora_adapters_error::LlamaLoraAdaptersError; -pub use llama_model_load_error::LlamaModelLoadError; -pub use logits_error::LogitsError; -pub use memory_seq_add_error::MemorySeqAddError; -pub use memory_seq_cp_error::MemorySeqCpError; -pub use memory_seq_div_error::MemorySeqDivError; -pub use memory_seq_pos_max_error::MemorySeqPosMaxError; -pub use memory_seq_rm_error::MemorySeqRmError; -pub use meta_val_error::MetaValError; -pub use model_params_error::ModelParamsError; -pub use new_llama_chat_message_error::NewLlamaChatMessageError; -pub use paired_quote_failure::PairedQuoteFailure; -pub use parse_chat_message_error::ParseChatMessageError; -pub use process_failure::ProcessFailure; -pub use sample_error::SampleError; -pub use sampler_accept_error::SamplerAcceptError; -pub use sampler_apply_error::SamplerApplyError; -pub use sampling_error::SamplingError; -pub use string_to_token_error::StringToTokenError; -pub use token_attr_error::TokenAttrError; -pub use token_outside_vocabulary::TokenOutsideVocabulary; -pub use token_to_string_error::TokenToStringError; -pub use tool_call_format_failure::ToolCallFormatFailure; -pub use xml_function_tags_failure::XmlFunctionTagsFailure; +use llama_cpp_error::LlamaCppError; pub type Result = std::result::Result; diff --git a/llama-cpp-bindings/src/error/batch_transfer_error.rs b/llama-cpp-bindings/src/error/batch_transfer_error.rs index eb100b69b..69787146d 100644 --- a/llama-cpp-bindings/src/error/batch_transfer_error.rs +++ b/llama-cpp-bindings/src/error/batch_transfer_error.rs @@ -3,9 +3,9 @@ use crate::context::context_limits::ContextLimits; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum BatchTransferError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("the batch was built for context limits {batch:?}, but the context has {context:?}")] LimitsMismatch { batch: ContextLimits, diff --git a/llama-cpp-bindings/src/error/chat_session_error.rs b/llama-cpp-bindings/src/error/chat_session_error.rs index f8355a8c4..30426a380 100644 --- a/llama-cpp-bindings/src/error/chat_session_error.rs +++ b/llama-cpp-bindings/src/error/chat_session_error.rs @@ -9,9 +9,9 @@ use crate::error::string_to_token_error::StringToTokenError; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum ChatSessionError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("the model's chat templates are unavailable: {0}")] Templates(#[from] ChatTemplatesError), #[error("a chat message is not valid UTF-8")] diff --git a/llama-cpp-bindings/src/error/chat_templates_error.rs b/llama-cpp-bindings/src/error/chat_templates_error.rs index eba331ad1..ac3cdada9 100644 --- a/llama-cpp-bindings/src/error/chat_templates_error.rs +++ b/llama-cpp-bindings/src/error/chat_templates_error.rs @@ -1,9 +1,9 @@ #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum ChatTemplatesError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("the model has no chat template")] ModelHasNoChatTemplate, #[error("not enough memory")] diff --git a/llama-cpp-bindings/src/error/decode_error.rs b/llama-cpp-bindings/src/error/decode_error.rs index 272bc5702..0f17fd1e5 100644 --- a/llama-cpp-bindings/src/error/decode_error.rs +++ b/llama-cpp-bindings/src/error/decode_error.rs @@ -43,7 +43,7 @@ impl From for DecodeError { mod tests { use std::num::NonZeroI32; - use super::DecodeError; + use crate::error::decode_error::DecodeError; #[test] fn no_kv_cache_slot_maps_from_code_one() { diff --git a/llama-cpp-bindings/src/error/embeddings_error.rs b/llama-cpp-bindings/src/error/embeddings_error.rs index eddf1c1ad..db7e8b175 100644 --- a/llama-cpp-bindings/src/error/embeddings_error.rs +++ b/llama-cpp-bindings/src/error/embeddings_error.rs @@ -16,7 +16,7 @@ pub enum EmbeddingsError { "Token embeddings need a context with LLAMA_POOLING_TYPE_NONE, but it pools with {pooling_type:?}" )] TokenEmbeddingsRequireNonePooling { - pooling_type: crate::context::params::LlamaPoolingType, + pooling_type: crate::context::llama_pooling_type::LlamaPoolingType, }, #[error("Invalid embedding dimension: {0}")] InvalidEmbeddingDimension(#[source] std::num::TryFromIntError), @@ -24,7 +24,7 @@ pub enum EmbeddingsError { "NextN embeddings need a context with LLAMA_POOLING_TYPE_NONE, but it pools with {pooling_type:?}" )] NextnEmbeddingsRequireNonePooling { - pooling_type: crate::context::params::LlamaPoolingType, + pooling_type: crate::context::llama_pooling_type::LlamaPoolingType, }, #[error("NextN embeddings weren't enabled on this context")] NextnEmbeddingsNotEnabled, @@ -41,9 +41,9 @@ pub enum EmbeddingsError { #[error("llama.cpp failed while checking for a NextN hidden state: {message}")] NextnProbeReported { message: String }, #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error( "No NextN embedding exists for token {token_index}; it was not marked as an output of the last decoded batch" )] diff --git a/llama-cpp-bindings/src/error/encode_error.rs b/llama-cpp-bindings/src/error/encode_error.rs index 72113234b..4da67ef98 100644 --- a/llama-cpp-bindings/src/error/encode_error.rs +++ b/llama-cpp-bindings/src/error/encode_error.rs @@ -42,7 +42,7 @@ impl From for EncodeError { mod tests { use std::num::NonZeroI32; - use super::EncodeError; + use crate::error::encode_error::EncodeError; #[test] fn no_kv_cache_slot_maps_from_code_one() { diff --git a/llama-cpp-bindings/src/error/eval_multimodal_chunks_error.rs b/llama-cpp-bindings/src/error/eval_multimodal_chunks_error.rs index 560436ebc..24d6c5aa5 100644 --- a/llama-cpp-bindings/src/error/eval_multimodal_chunks_error.rs +++ b/llama-cpp-bindings/src/error/eval_multimodal_chunks_error.rs @@ -1,4 +1,4 @@ -use crate::mtmd::MtmdEvalError; +use crate::mtmd::mtmd_eval_error::MtmdEvalError; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum EvalMultimodalChunksError { diff --git a/llama-cpp-bindings/src/error/fit_error.rs b/llama-cpp-bindings/src/error/fit_error.rs index 3e6f35c38..7c45a6a05 100644 --- a/llama-cpp-bindings/src/error/fit_error.rs +++ b/llama-cpp-bindings/src/error/fit_error.rs @@ -1,9 +1,9 @@ #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] pub enum FitError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("no parameter combination fits available memory")] NoFittingMemoryLayout, #[error("parameter fitting aborted")] diff --git a/llama-cpp-bindings/src/error/grammar_error.rs b/llama-cpp-bindings/src/error/grammar_error.rs index bd273ef0a..22e2babde 100644 --- a/llama-cpp-bindings/src/error/grammar_error.rs +++ b/llama-cpp-bindings/src/error/grammar_error.rs @@ -1,14 +1,14 @@ use std::ffi::NulError; -use crate::SamplingError; +use crate::error::sampling_error::SamplingError; use crate::error::token_to_string_error::TokenToStringError; #[derive(Debug, thiserror::Error, PartialEq, Eq)] pub enum GrammarError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("sampler initialization failed: {0}")] SamplerInitialization(#[from] SamplingError), #[error("the approximate token environment could not be built: {0}")] diff --git a/llama-cpp-bindings/src/error/json_schema_to_grammar_error.rs b/llama-cpp-bindings/src/error/json_schema_to_grammar_error.rs index aed9048af..5618ca78e 100644 --- a/llama-cpp-bindings/src/error/json_schema_to_grammar_error.rs +++ b/llama-cpp-bindings/src/error/json_schema_to_grammar_error.rs @@ -4,9 +4,9 @@ use std::string::FromUtf8Error; #[derive(Debug, PartialEq, Eq, thiserror::Error)] pub enum JsonSchemaToGrammarError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("schema string contains an interior NUL byte: {0}")] SchemaContainsNulByte(#[from] NulError), #[error("JSON schema is invalid: {message}")] diff --git a/llama-cpp-bindings/src/error/llama_context_load_error.rs b/llama-cpp-bindings/src/error/llama_context_load_error.rs index b0145ce37..b571d7aae 100644 --- a/llama-cpp-bindings/src/error/llama_context_load_error.rs +++ b/llama-cpp-bindings/src/error/llama_context_load_error.rs @@ -1,9 +1,9 @@ #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum LlamaContextLoadError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("context could not be constructed")] Unconstructible, #[error("not enough memory")] diff --git a/llama-cpp-bindings/src/error/llama_model_load_error.rs b/llama-cpp-bindings/src/error/llama_model_load_error.rs index 38b676d93..b9f69b616 100644 --- a/llama-cpp-bindings/src/error/llama_model_load_error.rs +++ b/llama-cpp-bindings/src/error/llama_model_load_error.rs @@ -4,9 +4,9 @@ use std::path::PathBuf; #[derive(Debug, PartialEq, Eq, thiserror::Error)] pub enum LlamaModelLoadError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("null byte in string {0}")] NullError(#[from] NulError), #[error("failed to convert path {0} to str")] diff --git a/llama-cpp-bindings/src/error/memory_seq_add_error.rs b/llama-cpp-bindings/src/error/memory_seq_add_error.rs index f244bb4b9..0f6aa2b47 100644 --- a/llama-cpp-bindings/src/error/memory_seq_add_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_add_error.rs @@ -3,9 +3,9 @@ use std::num::TryFromIntError; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum MemorySeqAddError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("provided start position is too large for an i32")] P0TooLarge(#[source] TryFromIntError), #[error("provided end position is too large for an i32")] diff --git a/llama-cpp-bindings/src/error/memory_seq_div_error.rs b/llama-cpp-bindings/src/error/memory_seq_div_error.rs index 61e22b699..e4b35ed03 100644 --- a/llama-cpp-bindings/src/error/memory_seq_div_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_div_error.rs @@ -3,9 +3,9 @@ use std::num::TryFromIntError; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum MemorySeqDivError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("provided start position is too large for an i32")] P0TooLarge(#[source] TryFromIntError), #[error("provided end position is too large for an i32")] diff --git a/llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs b/llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs index e281a077b..eed84ba53 100644 --- a/llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs @@ -1,9 +1,9 @@ #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum MemorySeqPosMaxError { #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error("context has no memory module available")] MemoryHandleUnavailable, #[error("sequence id {seq_id} is outside the context sequence range")] diff --git a/llama-cpp-bindings/src/error/parse_chat_message_error.rs b/llama-cpp-bindings/src/error/parse_chat_message_error.rs index cc0fe73be..d3c4fb2c4 100644 --- a/llama-cpp-bindings/src/error/parse_chat_message_error.rs +++ b/llama-cpp-bindings/src/error/parse_chat_message_error.rs @@ -3,9 +3,9 @@ use std::str::Utf8Error; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum ParseChatMessageError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/error/process_failure.rs b/llama-cpp-bindings/src/error/process_failure.rs index 73e5ba4f1..b557ef4c0 100644 --- a/llama-cpp-bindings/src/error/process_failure.rs +++ b/llama-cpp-bindings/src/error/process_failure.rs @@ -2,9 +2,9 @@ #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum ProcessFailure { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/error/sample_error.rs b/llama-cpp-bindings/src/error/sample_error.rs index 8fe7da983..b9467080e 100644 --- a/llama-cpp-bindings/src/error/sample_error.rs +++ b/llama-cpp-bindings/src/error/sample_error.rs @@ -5,9 +5,9 @@ use crate::error::sampler_apply_error::SamplerApplyError; #[derive(Debug, thiserror::Error, PartialEq, Eq)] pub enum SampleError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("the logits to sample from are unavailable: {0}")] Logits(#[from] LogitsError), #[error("not enough memory")] diff --git a/llama-cpp-bindings/src/error/sampler_accept_error.rs b/llama-cpp-bindings/src/error/sampler_accept_error.rs index b8865984a..81b68f310 100644 --- a/llama-cpp-bindings/src/error/sampler_accept_error.rs +++ b/llama-cpp-bindings/src/error/sampler_accept_error.rs @@ -1,9 +1,9 @@ #[derive(Debug, PartialEq, Eq, thiserror::Error)] pub enum SamplerAcceptError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/error/sampler_apply_error.rs b/llama-cpp-bindings/src/error/sampler_apply_error.rs index e7d5c1428..4cab8977f 100644 --- a/llama-cpp-bindings/src/error/sampler_apply_error.rs +++ b/llama-cpp-bindings/src/error/sampler_apply_error.rs @@ -1,9 +1,9 @@ #[derive(Debug, thiserror::Error, Clone, PartialEq, Eq)] pub enum SamplerApplyError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("the sampler pointer was null when applying to the token data array")] NullSampler, #[error("the sampler excluded every candidate, leaving no token to select")] diff --git a/llama-cpp-bindings/src/error/sampling_error.rs b/llama-cpp-bindings/src/error/sampling_error.rs index 4f6718799..c22991e31 100644 --- a/llama-cpp-bindings/src/error/sampling_error.rs +++ b/llama-cpp-bindings/src/error/sampling_error.rs @@ -3,9 +3,9 @@ use std::ffi::NulError; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum SamplingError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("a value does not fit into i32")] IntegerOverflow(#[source] std::num::TryFromIntError), #[error("the llama.cpp library ran out of memory initializing the {sampler} sampler")] diff --git a/llama-cpp-bindings/src/error/string_to_token_error.rs b/llama-cpp-bindings/src/error/string_to_token_error.rs index 8d9e5dd08..41ed7a5e0 100644 --- a/llama-cpp-bindings/src/error/string_to_token_error.rs +++ b/llama-cpp-bindings/src/error/string_to_token_error.rs @@ -3,9 +3,9 @@ use std::ffi::NulError; #[derive(Debug, PartialEq, Eq, thiserror::Error)] pub enum StringToTokenError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("{0}")] NulError(#[from] NulError), #[error("{0}")] diff --git a/llama-cpp-bindings/src/json_schema_to_grammar.rs b/llama-cpp-bindings/src/json_schema_to_grammar.rs index 2d3434d7c..b030f9e83 100644 --- a/llama-cpp-bindings/src/json_schema_to_grammar.rs +++ b/llama-cpp-bindings/src/json_schema_to_grammar.rs @@ -1,6 +1,6 @@ use std::ffi::{CStr, CString, c_char}; -use crate::error::JsonSchemaToGrammarError; +use crate::error::json_schema_to_grammar_error::JsonSchemaToGrammarError; use llama_cpp_ffi_status::read_and_free_cpp_string; /// # Safety @@ -17,7 +17,7 @@ unsafe fn json_schema_to_grammar_status_to_result( match status { llama_cpp_bindings_sys::LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_OK => { if out.is_null() { - return Err(crate::FfiContractError { + return Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "success status contained a null grammar string", } @@ -64,27 +64,27 @@ unsafe fn json_schema_to_grammar_status_to_result( Err(JsonSchemaToGrammarError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_NULL_SCHEMA_JSON_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "was given a null schema_json argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_NULL_OUT_GRAMMAR_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "was given a null out_grammar argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "was given a null out_error argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_json_schema_to_grammar", code: i64::from(other), } @@ -114,8 +114,8 @@ pub fn json_schema_to_grammar(schema_json: &str) -> Result { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: OPERATION, detail: "was given a null text argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_OUT_STRING_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: OPERATION, detail: "was given a null out_string argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: OPERATION, detail: "was given a null out_error argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: OPERATION, code: i64::from(other), } @@ -145,10 +145,12 @@ mod tests { regex_escape_status_to_result(status, out_string, out_error) }; let contract = |detail| { - Err(GrammarError::FfiContract(crate::FfiContractError { - operation: "llama_rs_regex_escape", - detail, - })) + Err(GrammarError::FfiContract( + llama_cpp_ffi_status::FfiContractError { + operation: "llama_rs_regex_escape", + detail, + }, + )) }; assert_eq!( @@ -208,10 +210,12 @@ mod tests { contract("was given a null text argument"), contract("was given a null out_string argument"), contract("was given a null out_error argument"), - Err(GrammarError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_regex_escape", - code: 255 - })), + Err(GrammarError::FfiStatus( + llama_cpp_ffi_status::FfiStatusError { + operation: "llama_rs_regex_escape", + code: 255 + } + )), ] ); } @@ -224,10 +228,12 @@ mod tests { ptr::null_mut(), ptr::null_mut() ), - Err(GrammarError::FfiContract(crate::FfiContractError { - operation: "llama_rs_regex_escape", - detail: "reported a thrown C++ exception without an error message" - })) + Err(GrammarError::FfiContract( + llama_cpp_ffi_status::FfiContractError { + operation: "llama_rs_regex_escape", + detail: "reported a thrown C++ exception without an error message" + } + )) ); } diff --git a/llama-cpp-bindings/src/lib.rs b/llama-cpp-bindings/src/lib.rs index e7ec77cd6..cc8dc5e3b 100644 --- a/llama-cpp-bindings/src/lib.rs +++ b/llama-cpp-bindings/src/lib.rs @@ -77,51 +77,3 @@ pub mod streaming_json_probe; pub mod token; pub mod token_piece; pub mod tool_call_format; - -pub use error::{ - ChatSessionError, ChatTemplateError, ChatTemplatesError, DecodeError, EmbeddingsError, - EncodeError, EvalMultimodalChunksError, FfiContractError, FfiStatusError, GrammarError, - IngestError, JsonSchemaToGrammarError, LlamaContextLoadError, LlamaCppError, - LlamaLoraAdapterInitError, LlamaLoraAdaptersError, LlamaModelLoadError, LogitsError, - MemorySeqAddError, MemorySeqCpError, MemorySeqDivError, MemorySeqPosMaxError, MemorySeqRmError, - MetaValError, ModelParamsError, NewLlamaChatMessageError, ParseChatMessageError, - ProcessFailure, Result, SampleError, SamplerAcceptError, SamplingError, StringToTokenError, - TokenAttrError, TokenOutsideVocabulary, TokenToStringError, -}; - -pub use bare_json_tool_calls::BareJsonToolCalls; -pub use chat_assistant_prefill::ChatAssistantPrefill; -pub use chat_assistant_turn::ChatAssistantTurn; -pub use chat_classification::ChatClassification; -pub use chat_grammar_trigger::ChatGrammarTrigger; -pub use chat_message_parse_outcome::ChatMessageParseOutcome; -pub use chat_output::ChatOutput; -pub use chat_output_completeness::ChatOutputCompleteness; -pub use chat_prompt_part::ChatPromptPart; -pub use chat_session::ChatSession; -pub use chat_session_options::ChatSessionOptions; -pub use chat_templates::ChatTemplates; -pub use chat_tool_choice::ChatToolChoice; -pub use classified_sample::ClassifiedSample; -pub use eval_multimodal_chunks_params::EvalMultimodalChunksParams; -pub use generation_progress::GenerationProgress; -pub use llama_backend_device::LlamaBackendDevice; -pub use llama_backend_device_type::LlamaBackendDeviceType; -pub use llama_cpp_bindings_types::{ - BracketedJsonShape, KeyValueXmlTagsShape, PairedQuoteShape, ParsedChatMessage, ParsedToolCall, - ParsedToolCallId, TokenUsage, TokenUsageError, ToolCallArgsShape, ToolCallArguments, - ToolCallMarkers, ToolCallValueQuote, XmlTagsShape, -}; -pub use llama_cpp_gbnf::gbnf_validation_error::GbnfValidationError; -pub use raw_chat_message::RawChatMessage; -pub use sampled_token::SampledToken; -pub use sampled_token_classifier::SampledTokenClassifier; -pub use sampled_token_section::SampledTokenSection; -pub use token_piece::TokenPiece; - -pub use json_schema_to_grammar::json_schema_to_grammar; -pub use llama_time_us::llama_time_us; -pub use max_devices::max_devices; - -pub use log_options::LogOptions; -pub use send_logs_to_log::send_logs_to_log; diff --git a/llama-cpp-bindings/src/llama_backend.rs b/llama-cpp-bindings/src/llama_backend.rs index 5a15be466..fdec72e4c 100644 --- a/llama-cpp-bindings/src/llama_backend.rs +++ b/llama-cpp-bindings/src/llama_backend.rs @@ -1,7 +1,7 @@ use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering::SeqCst; -use crate::LlamaCppError; +use crate::error::llama_cpp_error::LlamaCppError; use crate::llama_backend_numa_strategy::NumaStrategy; #[derive(Eq, PartialEq, Debug)] @@ -10,7 +10,7 @@ pub struct LlamaBackend {} static LLAMA_BACKEND_INITIALIZED: AtomicBool = AtomicBool::new(false); impl LlamaBackend { - fn mark_init() -> crate::Result<()> { + fn mark_init() -> crate::error::Result<()> { match LLAMA_BACKEND_INITIALIZED.compare_exchange(false, true, SeqCst, SeqCst) { Ok(_was_uninitialized) => Ok(()), Err(_was_already_initialized) => Err(LlamaCppError::BackendAlreadyInitialized), @@ -19,7 +19,7 @@ impl LlamaBackend { /// # Errors /// Returns an error if the backend was already initialized. - pub fn init() -> crate::Result { + pub fn init() -> crate::error::Result { Self::mark_init()?; unsafe { llama_cpp_bindings_sys::llama_backend_init() } Ok(Self {}) @@ -30,7 +30,7 @@ impl LlamaBackend { /// /// # Errors /// Returns an error if the backend was already initialized. - pub fn init_numa(strategy: NumaStrategy) -> crate::Result { + pub fn init_numa(strategy: NumaStrategy) -> crate::error::Result { let backend = Self::init()?; unsafe { @@ -70,7 +70,7 @@ mod tests { use serial_test::serial; use super::LlamaBackend; - use crate::LlamaCppError; + use crate::error::llama_cpp_error::LlamaCppError; #[test] #[serial] diff --git a/llama-cpp-bindings/src/llama_backend_device.rs b/llama-cpp-bindings/src/llama_backend_device.rs index b1108ed17..2a09daee5 100644 --- a/llama-cpp-bindings/src/llama_backend_device.rs +++ b/llama-cpp-bindings/src/llama_backend_device.rs @@ -1,6 +1,6 @@ +use crate::llama_backend_device_type::LlamaBackendDeviceType; use std::ffi::c_char; -pub use crate::llama_backend_device_type::LlamaBackendDeviceType; fn cstr_to_string(ptr: *const c_char) -> String { if ptr.is_null() { String::new() @@ -61,8 +61,8 @@ impl LlamaBackendDevice { #[cfg(test)] mod tests { - use super::LlamaBackendDevice; use super::cstr_to_string; + use crate::llama_backend_device::LlamaBackendDevice; #[test] fn cstr_to_string_with_null_returns_empty() { diff --git a/llama-cpp-bindings/src/llama_backend_device_type.rs b/llama-cpp-bindings/src/llama_backend_device_type.rs index c1f971e70..00e51f175 100644 --- a/llama-cpp-bindings/src/llama_backend_device_type.rs +++ b/llama-cpp-bindings/src/llama_backend_device_type.rs @@ -22,7 +22,7 @@ impl LlamaBackendDeviceType { #[cfg(test)] mod tests { - use super::LlamaBackendDeviceType; + use crate::llama_backend_device_type::LlamaBackendDeviceType; #[test] fn device_type_from_raw_all_variants() { diff --git a/llama-cpp-bindings/src/llama_time_us.rs b/llama-cpp-bindings/src/llama_time_us.rs index 63d43ad8c..1a5626fc7 100644 --- a/llama-cpp-bindings/src/llama_time_us.rs +++ b/llama-cpp-bindings/src/llama_time_us.rs @@ -5,7 +5,7 @@ pub fn llama_time_us() -> i64 { #[cfg(test)] mod tests { - use super::llama_time_us; + use crate::llama_time_us::llama_time_us; #[test] fn returns_positive_value() { diff --git a/llama-cpp-bindings/src/llama_token_attrs.rs b/llama-cpp-bindings/src/llama_token_attrs.rs index 00c6f3cff..b699d1098 100644 --- a/llama-cpp-bindings/src/llama_token_attrs.rs +++ b/llama-cpp-bindings/src/llama_token_attrs.rs @@ -38,7 +38,9 @@ impl TryFrom for LlamaTokenAttrs { mod tests { use enumflags2::BitFlags; - use super::{LlamaTokenAttr, LlamaTokenAttrs, LlamaTokenAttrsFromIntError}; + use super::LlamaTokenAttr; + use super::LlamaTokenAttrs; + use super::LlamaTokenAttrsFromIntError; #[test] fn try_from_valid_single_attribute() { diff --git a/llama-cpp-bindings/src/llguidance_sampler.rs b/llama-cpp-bindings/src/llguidance_sampler.rs index aec1e71e5..89994fa84 100644 --- a/llama-cpp-bindings/src/llguidance_sampler.rs +++ b/llama-cpp-bindings/src/llguidance_sampler.rs @@ -7,7 +7,7 @@ use llguidance::earley::SlicedBiasComputer; use toktrie::ApproximateTokEnv; use toktrie::InferenceCapabilities; -use crate::GrammarError; +use crate::error::grammar_error::GrammarError; use crate::grammar_matcher::GrammarMatcher; use crate::mask_outcome::MaskOutcome; use crate::model::LlamaModel; @@ -197,10 +197,10 @@ mod tests { use super::UNCOMPILABLE_BIAS_SLICE; use super::create_llg_sampler_from_tok_env; - use crate::GrammarError; - use crate::SampleError; - use crate::SamplerAcceptError; + use crate::error::grammar_error::GrammarError; use crate::error::grammar_runtime_error::GrammarRuntimeError; + use crate::error::sample_error::SampleError; + use crate::error::sampler_accept_error::SamplerAcceptError; use crate::sampling::LlamaSampler; use crate::token::LlamaToken; use crate::token::data::LlamaTokenData; diff --git a/llama-cpp-bindings/src/log_options.rs b/llama-cpp-bindings/src/log_options.rs index 6192d0fff..effa910ab 100644 --- a/llama-cpp-bindings/src/log_options.rs +++ b/llama-cpp-bindings/src/log_options.rs @@ -22,7 +22,7 @@ impl LogOptions { #[cfg(test)] mod tests { - use super::LogOptions; + use crate::log_options::LogOptions; #[test] fn builder_chain_sets_both_flags() { diff --git a/llama-cpp-bindings/src/max_devices.rs b/llama-cpp-bindings/src/max_devices.rs index 014eeaee1..910225d59 100644 --- a/llama-cpp-bindings/src/max_devices.rs +++ b/llama-cpp-bindings/src/max_devices.rs @@ -5,7 +5,7 @@ pub fn max_devices() -> usize { #[cfg(test)] mod tests { - use super::max_devices; + use crate::max_devices::max_devices; #[test] fn returns_at_least_one() { diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index 8eda66824..6fde4fe1e 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -31,30 +31,28 @@ use toktrie::TokRxInfo; use toktrie::TokTrie; use crate::chat_templates::ChatTemplates; +use crate::error::chat_template_error::ChatTemplateError; use crate::error::chat_templates_error::ChatTemplatesError; +use crate::error::llama_lora_adapter_init_error::LlamaLoraAdapterInitError; +use crate::error::llama_model_load_error::LlamaModelLoadError; +use crate::error::meta_val_error::MetaValError; +use crate::error::string_to_token_error::StringToTokenError; +use crate::error::token_attr_error::TokenAttrError; +use crate::error::token_outside_vocabulary::TokenOutsideVocabulary; +use crate::error::token_to_string_error::TokenToStringError; use crate::llama_backend::LlamaBackend; use crate::llama_token_attrs::LlamaTokenAttrs; use crate::model::tokenizer_input::TokenizerInput; use crate::sampled_token::SampledToken; use crate::token::LlamaToken; -use crate::{ - ChatTemplateError, LlamaLoraAdapterInitError, LlamaModelLoadError, MetaValError, - StringToTokenError, TokenAttrError, TokenOutsideVocabulary, TokenToStringError, -}; - -pub use add_bos::AddBos; -pub use llama_chat_message::LlamaChatMessage; -pub use llama_chat_template::LlamaChatTemplate; -pub use llama_lazy_mode::LlamaLazyMode; -pub use llama_lazy_mode_parse_error::LlamaLazyModeParseError; -pub use llama_load_mode::LlamaLoadMode; -pub use llama_load_mode_parse_error::LlamaLoadModeParseError; -pub use llama_lora_adapter::LlamaLoraAdapter; -pub use parse_special_tokens::ParseSpecialTokens; -pub use rope_type::RopeType; -pub use vocab_type::VocabType; -pub use vocab_type_from_int_error::VocabTypeFromIntError; +use crate::model::add_bos::AddBos; +use crate::model::llama_chat_template::LlamaChatTemplate; +use crate::model::llama_lora_adapter::LlamaLoraAdapter; +use crate::model::parse_special_tokens::ParseSpecialTokens; +use crate::model::rope_type::RopeType; +use crate::model::vocab_type::VocabType; +use crate::model::vocab_type_from_int_error::VocabTypeFromIntError; use llama_cpp_ffi_status::read_and_free_cpp_string; use params::LlamaModelParams; @@ -116,7 +114,7 @@ unsafe fn load_model_from_file_status_to_result( match status { llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_OK => { let model = NonNull::new(out_model).ok_or_else(|| { - LlamaModelLoadError::from(crate::FfiContractError { + LlamaModelLoadError::from(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_load_model_from_file", detail: "success status contained a null model", }) @@ -151,27 +149,27 @@ unsafe fn load_model_from_file_status_to_result( Err(LlamaModelLoadError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_NULL_PATH_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_load_model_from_file", detail: "was given a null path argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_NULL_OUT_MODEL_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_load_model_from_file", detail: "was given a null out_model argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_load_model_from_file", detail: "was given a null out_error argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_load_model_from_file", code: i64::from(other), } @@ -728,7 +726,7 @@ unsafe fn tokenize_status_to_result( } llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_VOCAB_ARG => { unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_tokenize", detail: "was given a null vocab argument", } @@ -736,7 +734,7 @@ unsafe fn tokenize_status_to_result( } llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_TEXT_ARG => { unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_tokenize", detail: "was given a null text argument", } @@ -744,7 +742,7 @@ unsafe fn tokenize_status_to_result( } llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_OUT_RETURNED_COUNT_ARG => { unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_tokenize", detail: "was given a null out_returned_count argument", } @@ -752,7 +750,7 @@ unsafe fn tokenize_status_to_result( } llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_OUT_ERROR_ARG => { unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_tokenize", detail: "was given a null out_error argument", } @@ -760,7 +758,7 @@ unsafe fn tokenize_status_to_result( } other => { unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiStatusError { + Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_tokenize", code: i64::from(other), } @@ -883,7 +881,7 @@ mod cstring_with_validated_len_tests { use std::str; use super::cstring_with_validated_len; - use crate::StringToTokenError; + use crate::error::string_to_token_error::StringToTokenError; #[test] fn cstring_with_validated_len_null_byte_returns_error() { @@ -916,7 +914,9 @@ mod validate_string_length_for_tokenizer_tests { fn validate_string_length_overflow_returns_error() { assert_eq!( validate_string_length_for_tokenizer(usize::MAX), - i32::try_from(usize::MAX).map_err(crate::StringToTokenError::CIntConversionError) + i32::try_from(usize::MAX).map_err( + crate::error::string_to_token_error::StringToTokenError::CIntConversionError + ) ); } } @@ -980,10 +980,13 @@ mod tokenize_into_buffer_tests { #[test] fn tokenize_into_buffer_propagates_invocation_error() { let result = tokenize_into_buffer(8, |_tokens, _n_tokens_max| { - Err(crate::StringToTokenError::NotEnoughMemory) + Err(crate::error::string_to_token_error::StringToTokenError::NotEnoughMemory) }); - assert_eq!(result, Err(crate::StringToTokenError::NotEnoughMemory)); + assert_eq!( + result, + Err(crate::error::string_to_token_error::StringToTokenError::NotEnoughMemory) + ); } #[test] @@ -995,11 +998,14 @@ mod tokenize_into_buffer_tests { if count == 0 { Ok(-20) } else { - Err(crate::StringToTokenError::NotEnoughMemory) + Err(crate::error::string_to_token_error::StringToTokenError::NotEnoughMemory) } }); - assert_eq!(result, Err(crate::StringToTokenError::NotEnoughMemory)); + assert_eq!( + result, + Err(crate::error::string_to_token_error::StringToTokenError::NotEnoughMemory) + ); assert_eq!(call_count.get(), 2); } @@ -1007,9 +1013,11 @@ mod tokenize_into_buffer_tests { fn tokenize_into_buffer_refuses_a_capacity_beyond_a_c_int_before_allocating() { assert_eq!( tokenize_into_buffer(usize::MAX, |_tokens, _n_tokens_max| Ok(0)), - Err(crate::StringToTokenError::CIntConversionError( - i32::try_from(usize::MAX).unwrap_err() - )) + Err( + crate::error::string_to_token_error::StringToTokenError::CIntConversionError( + i32::try_from(usize::MAX).unwrap_err() + ) + ) ); } @@ -1017,9 +1025,11 @@ mod tokenize_into_buffer_tests { fn tokenize_into_buffer_refuses_a_required_size_it_cannot_negate() { assert_eq!( tokenize_into_buffer(8, |_tokens, _n_tokens_max| Ok(i32::MIN)), - Err(crate::StringToTokenError::CIntConversionError( - usize::try_from(i32::MIN).unwrap_err() - )) + Err( + crate::error::string_to_token_error::StringToTokenError::CIntConversionError( + usize::try_from(i32::MIN).unwrap_err() + ) + ) ); } @@ -1034,7 +1044,9 @@ mod tokenize_into_buffer_tests { assert_eq!( result.unwrap_err(), - crate::StringToTokenError::CIntConversionError(usize::try_from(-5i32).unwrap_err()) + crate::error::string_to_token_error::StringToTokenError::CIntConversionError( + usize::try_from(-5i32).unwrap_err() + ) ); } } @@ -1042,7 +1054,7 @@ mod tokenize_into_buffer_tests { #[cfg(test)] mod extract_meta_string_tests { use super::extract_meta_string; - use crate::MetaValError; + use crate::error::meta_val_error::MetaValError; #[test] fn returns_error_when_null_terminator_missing() { @@ -1120,7 +1132,7 @@ mod initialize_lora_adapter_tests { use std::ptr; use super::initialize_lora_adapter; - use crate::LlamaLoraAdapterInitError; + use crate::error::llama_lora_adapter_init_error::LlamaLoraAdapterInitError; #[test] fn lora_adapter_initialization_maps_a_null_pointer_to_unloadable() { @@ -1147,7 +1159,7 @@ mod load_model_from_file_status_to_result_tests { use std::ptr; use super::load_model_from_file_status_to_result; - use crate::LlamaModelLoadError; + use crate::error::llama_model_load_error::LlamaModelLoadError; #[test] fn load_model_success_with_null_model_is_contract_error() { @@ -1162,7 +1174,7 @@ mod load_model_from_file_status_to_result_tests { assert_eq!( result.unwrap_err(), - LlamaModelLoadError::FfiContract(crate::FfiContractError { + LlamaModelLoadError::FfiContract(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_load_model_from_file", detail: "success status contained a null model", }) @@ -1236,7 +1248,7 @@ mod load_model_from_file_status_to_result_tests { assert_eq!( result.unwrap_err(), - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_load_model_from_file", detail: "reported a thrown C++ exception without an error message", } @@ -1257,7 +1269,7 @@ mod load_model_from_file_status_to_result_tests { assert_eq!( result.unwrap_err(), - LlamaModelLoadError::FfiStatus(crate::FfiStatusError { + LlamaModelLoadError::FfiStatus(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_load_model_from_file", code: 255, }) @@ -1277,7 +1289,7 @@ mod load_model_from_file_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_load_model_from_file", detail: "was given a null path argument", } @@ -1299,7 +1311,7 @@ mod load_model_from_file_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_load_model_from_file", detail: "was given a null out_model argument", } @@ -1321,7 +1333,7 @@ mod load_model_from_file_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_load_model_from_file", detail: "was given a null out_error argument", } @@ -1352,7 +1364,7 @@ mod tokenize_status_to_result_tests { use std::ptr; use super::tokenize_status_to_result; - use crate::StringToTokenError; + use crate::error::string_to_token_error::StringToTokenError; #[test] fn tokenize_ok_returns_count() { @@ -1412,7 +1424,7 @@ mod tokenize_status_to_result_tests { assert_eq!( result, - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_tokenize", detail: "reported a thrown C++ exception without an error message", } @@ -1426,10 +1438,12 @@ mod tokenize_status_to_result_tests { assert_eq!( result, - Err(StringToTokenError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_tokenize", - code: 255, - })) + Err(StringToTokenError::FfiStatus( + llama_cpp_ffi_status::FfiStatusError { + operation: "llama_rs_tokenize", + code: 255, + } + )) ); } @@ -1445,7 +1459,7 @@ mod tokenize_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_tokenize", detail: "was given a null vocab argument", } @@ -1466,7 +1480,7 @@ mod tokenize_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_tokenize", detail: "was given a null text argument", } @@ -1487,7 +1501,7 @@ mod tokenize_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_tokenize", detail: "was given a null out_returned_count argument", } @@ -1508,7 +1522,7 @@ mod tokenize_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_tokenize", detail: "was given a null out_error argument", } diff --git a/llama-cpp-bindings/src/model/llama_chat_message.rs b/llama-cpp-bindings/src/model/llama_chat_message.rs index 51e1f0867..6a1ca8dfd 100644 --- a/llama-cpp-bindings/src/model/llama_chat_message.rs +++ b/llama-cpp-bindings/src/model/llama_chat_message.rs @@ -1,6 +1,6 @@ use std::ffi::CString; -use crate::NewLlamaChatMessageError; +use crate::error::new_llama_chat_message_error::NewLlamaChatMessageError; #[derive(Debug, Eq, PartialEq, Clone)] pub struct LlamaChatMessage { @@ -21,7 +21,7 @@ impl LlamaChatMessage { #[cfg(test)] mod tests { - use super::LlamaChatMessage; + use crate::model::llama_chat_message::LlamaChatMessage; #[test] fn valid_construction() { diff --git a/llama-cpp-bindings/src/model/llama_chat_template.rs b/llama-cpp-bindings/src/model/llama_chat_template.rs index b752b7bd5..56ead4f14 100644 --- a/llama-cpp-bindings/src/model/llama_chat_template.rs +++ b/llama-cpp-bindings/src/model/llama_chat_template.rs @@ -37,7 +37,7 @@ impl std::fmt::Debug for LlamaChatTemplate { #[cfg(test)] mod tests { - use super::LlamaChatTemplate; + use crate::model::llama_chat_template::LlamaChatTemplate; #[test] fn valid_template_creation() { diff --git a/llama-cpp-bindings/src/model/llama_lazy_mode.rs b/llama-cpp-bindings/src/model/llama_lazy_mode.rs index c3745c784..f234acfab 100644 --- a/llama-cpp-bindings/src/model/llama_lazy_mode.rs +++ b/llama-cpp-bindings/src/model/llama_lazy_mode.rs @@ -35,7 +35,7 @@ impl From for llama_cpp_bindings_sys::llama_lazy_mode { #[cfg(test)] mod tests { - use super::LlamaLazyMode; + use crate::model::llama_lazy_mode::LlamaLazyMode; use crate::model::llama_lazy_mode_parse_error::LlamaLazyModeParseError; const LAZY_MODES: &[(LlamaLazyMode, llama_cpp_bindings_sys::llama_lazy_mode)] = &[ diff --git a/llama-cpp-bindings/src/model/llama_load_mode.rs b/llama-cpp-bindings/src/model/llama_load_mode.rs index 546175103..73618744e 100644 --- a/llama-cpp-bindings/src/model/llama_load_mode.rs +++ b/llama-cpp-bindings/src/model/llama_load_mode.rs @@ -41,7 +41,7 @@ impl From for llama_cpp_bindings_sys::llama_load_mode { #[cfg(test)] mod tests { - use super::LlamaLoadMode; + use crate::model::llama_load_mode::LlamaLoadMode; const LOAD_MODES: &[(LlamaLoadMode, llama_cpp_bindings_sys::llama_load_mode)] = &[ ( @@ -91,7 +91,11 @@ mod tests { fn unknown_ffi_load_mode_preserves_its_value() { assert_eq!( LlamaLoadMode::try_from(i32::MAX), - Err(super::LlamaLoadModeParseError { value: i32::MAX }) + Err( + crate::model::llama_load_mode_parse_error::LlamaLoadModeParseError { + value: i32::MAX + } + ) ); } } diff --git a/llama-cpp-bindings/src/model/llama_lora_adapter.rs b/llama-cpp-bindings/src/model/llama_lora_adapter.rs index d0a4484c4..99e873eda 100644 --- a/llama-cpp-bindings/src/model/llama_lora_adapter.rs +++ b/llama-cpp-bindings/src/model/llama_lora_adapter.rs @@ -40,7 +40,7 @@ mod ownership_tests { use std::mem::ManuallyDrop; use std::ptr::NonNull; - use super::LlamaLoraAdapter; + use crate::model::llama_lora_adapter::LlamaLoraAdapter; #[test] fn adapter_preserves_the_owned_native_pointer() { diff --git a/llama-cpp-bindings/src/model/lora_adapter_scale.rs b/llama-cpp-bindings/src/model/lora_adapter_scale.rs index beef995b7..0add5e957 100644 --- a/llama-cpp-bindings/src/model/lora_adapter_scale.rs +++ b/llama-cpp-bindings/src/model/lora_adapter_scale.rs @@ -1,4 +1,4 @@ -use crate::model::LlamaLoraAdapter; +use crate::model::llama_lora_adapter::LlamaLoraAdapter; #[derive(Debug)] pub struct LoraAdapterScale<'adapter, 'model> { diff --git a/llama-cpp-bindings/src/model/params.rs b/llama-cpp-bindings/src/model/params.rs index 65529a1cb..6c963acd0 100644 --- a/llama-cpp-bindings/src/model/params.rs +++ b/llama-cpp-bindings/src/model/params.rs @@ -3,10 +3,11 @@ use std::fmt::{Debug, Formatter}; use std::pin::Pin; use std::ptr::null; -use crate::LlamaCppError; use crate::backend_device_index::backend_device_index; use crate::context::params::LlamaContextParams; -use crate::error::{FitError, ModelParamsError}; +use crate::error::fit_error::FitError; +use crate::error::llama_cpp_error::LlamaCppError; +use crate::error::model_params_error::ModelParamsError; use crate::model::llama_lazy_mode::LlamaLazyMode; use crate::model::llama_lazy_mode_parse_error::LlamaLazyModeParseError; use crate::model::llama_load_mode::LlamaLoadMode; @@ -63,62 +64,62 @@ fn fit_params_status_to_result( Err(FitError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_PATH_MODEL_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_fit_params", detail: "was given a null path_model argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_MPARAMS_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_fit_params", detail: "was given a null mparams argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_CPARAMS_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_fit_params", detail: "was given a null cparams argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_OUT_UNRECOGNIZED_STATUS_CODE_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_fit_params", detail: "was given a null out_unrecognized_status_code argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_fit_params", detail: "was given a null out_error argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_PATH_MODEL_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_fit_params", detail: "was given an extra model with a null path_model argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_MPARAMS_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_fit_params", detail: "was given an extra model with a null mparams argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_CPARAMS_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_fit_params", detail: "was given an extra model with a null cparams argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_fit_params", code: i64::from(other), } @@ -372,7 +373,7 @@ impl LlamaModelParams { for dev in self.devices.iter_mut() { *dev = std::ptr::null_mut(); } - let max_devices = crate::max_devices().min(LLAMA_CPP_MAX_DEVICES); + let max_devices = crate::max_devices::max_devices().min(LLAMA_CPP_MAX_DEVICES); if devices.len() > max_devices { return Err(LlamaCppError::MaxDevicesExceeded(max_devices)); } @@ -492,8 +493,8 @@ mod tests { use super::LLAMA_CPP_MAX_DEVICES; use super::LlamaModelParams; use super::fit_params_status_to_result; - use crate::LlamaCppError; use crate::error::fit_error::FitError; + use crate::error::llama_cpp_error::LlamaCppError; use crate::model::llama_lazy_mode::LlamaLazyMode; use crate::model::llama_lazy_mode_parse_error::LlamaLazyModeParseError; use crate::model::llama_load_mode::LlamaLoadMode; @@ -506,7 +507,7 @@ mod tests { } fn contract_error_with(detail: &'static str) -> FitError { - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_fit_params", detail, } @@ -515,7 +516,7 @@ mod tests { #[test] fn device_capacity_is_not_smaller_than_the_upstream_limit() { - let upstream_limit = crate::max_devices(); + let upstream_limit = crate::max_devices::max_devices(); assert!( LLAMA_CPP_MAX_DEVICES >= upstream_limit, "LLAMA_CPP_MAX_DEVICES ({LLAMA_CPP_MAX_DEVICES}) backs a fixed-size device array and is \ @@ -752,7 +753,7 @@ mod tests { assert_eq!( result.unwrap_err(), - crate::error::ModelParamsError::SlotNotEmpty + crate::error::model_params_error::ModelParamsError::SlotNotEmpty ); } @@ -783,7 +784,7 @@ mod tests { assert_eq!( result.unwrap_err(), - crate::error::ModelParamsError::SlotNotEmpty + crate::error::model_params_error::ModelParamsError::SlotNotEmpty ); } @@ -791,7 +792,7 @@ mod tests { fn with_devices_too_many_returns_max_exceeded() { let too_many: Vec = (0..17).collect(); let result = LlamaModelParams::default().with_devices(&too_many); - let device_limit = crate::max_devices().min(LLAMA_CPP_MAX_DEVICES); + let device_limit = crate::max_devices::max_devices().min(LLAMA_CPP_MAX_DEVICES); assert!( matches!( @@ -843,10 +844,12 @@ mod tests { assert_eq!( std::mem::discriminant(&result.unwrap_err()), - std::mem::discriminant(&crate::error::ModelParamsError::InvalidCharacterInKey { - byte: 0, - reason: String::new(), - }), + std::mem::discriminant( + &crate::error::model_params_error::ModelParamsError::InvalidCharacterInKey { + byte: 0, + reason: String::new(), + } + ), ); } @@ -859,10 +862,12 @@ mod tests { assert_eq!( std::mem::discriminant(&result.unwrap_err()), - std::mem::discriminant(&crate::error::ModelParamsError::InvalidCharacterInKey { - byte: 0, - reason: String::new(), - }), + std::mem::discriminant( + &crate::error::model_params_error::ModelParamsError::InvalidCharacterInKey { + byte: 0, + reason: String::new(), + } + ), ); } @@ -880,7 +885,7 @@ mod tests { assert_eq!( result.unwrap_err(), - crate::error::ModelParamsError::NoAvailableSlot + crate::error::model_params_error::ModelParamsError::NoAvailableSlot ); } @@ -894,7 +899,7 @@ mod tests { assert_eq!( result.unwrap_err(), - crate::error::ModelParamsError::NoAvailableSlot + crate::error::model_params_error::ModelParamsError::NoAvailableSlot ); } @@ -902,13 +907,13 @@ mod tests { #[serial_test::serial] fn fit_params_invalid_model_path_returns_error() { use crate::context::params::LlamaContextParams; - use crate::error::FitError; + use crate::error::fit_error::FitError; use crate::llama_backend::LlamaBackend; let _backend = LlamaBackend::init(); let mut params = std::pin::pin!(LlamaModelParams::default()); let mut context_params = LlamaContextParams::default(); - let mut margins = vec![0usize; crate::max_devices()]; + let mut margins = vec![0usize; crate::max_devices::max_devices()]; let bogus_path = c"/nonexistent/path/to/model.gguf"; let result = params.as_mut().fit_params( @@ -926,11 +931,11 @@ mod tests { #[test] fn fit_params_refuses_margins_not_sized_to_the_device_count() { use crate::context::params::LlamaContextParams; - use crate::error::FitError; + use crate::error::fit_error::FitError; let mut params = std::pin::pin!(LlamaModelParams::default()); let mut context_params = LlamaContextParams::default(); - let mut margins = vec![0usize; crate::max_devices() + 1]; + let mut margins = vec![0usize; crate::max_devices::max_devices() + 1]; let result = params.as_mut().fit_params( c"/nonexistent/path/to/model.gguf", @@ -944,8 +949,8 @@ mod tests { assert_eq!( result.err(), Some(FitError::MarginsDeviceCountMismatch { - expected: crate::max_devices(), - actual: crate::max_devices() + 1, + expected: crate::max_devices::max_devices(), + actual: crate::max_devices::max_devices() + 1, }) ); } @@ -1129,7 +1134,7 @@ mod fit_params_status_to_result_tests { assert_eq!( result, - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_fit_params", detail: "reported a thrown C++ exception without an error message", } @@ -1143,7 +1148,7 @@ mod fit_params_status_to_result_tests { assert_eq!( result, - Err(FitError::FfiStatus(crate::FfiStatusError { + Err(FitError::FfiStatus(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_fit_params", code: 255, })) diff --git a/llama-cpp-bindings/src/model/rope_type.rs b/llama-cpp-bindings/src/model/rope_type.rs index 26f4dd44c..e027c6ca7 100644 --- a/llama-cpp-bindings/src/model/rope_type.rs +++ b/llama-cpp-bindings/src/model/rope_type.rs @@ -23,7 +23,7 @@ impl RopeType { #[cfg(test)] mod tests { - use super::RopeType; + use crate::model::rope_type::RopeType; #[test] fn rope_type_none() { diff --git a/llama-cpp-bindings/src/model/vocab_type.rs b/llama-cpp-bindings/src/model/vocab_type.rs index eba208403..031ec7fd8 100644 --- a/llama-cpp-bindings/src/model/vocab_type.rs +++ b/llama-cpp-bindings/src/model/vocab_type.rs @@ -35,7 +35,8 @@ impl TryFrom for VocabType { #[cfg(test)] mod tests { - use super::{VocabType, VocabTypeFromIntError}; + use crate::model::vocab_type::VocabType; + use crate::model::vocab_type_from_int_error::VocabTypeFromIntError; #[test] fn every_llama_cpp_vocab_type_maps_to_its_variant() { diff --git a/llama-cpp-bindings/src/mtmd.rs b/llama-cpp-bindings/src/mtmd.rs index 1d511b834..0ef3d45e5 100644 --- a/llama-cpp-bindings/src/mtmd.rs +++ b/llama-cpp-bindings/src/mtmd.rs @@ -29,32 +29,3 @@ pub mod mtmd_memory_usage; pub mod mtmd_memory_usage_error; pub mod mtmd_tokenize_error; pub mod non_causal_chunk_micro_batch_mismatch; - -pub use fitted_mtmd_input_chunk::FittedMtmdInputChunk; -pub use fitted_mtmd_input_chunks::FittedMtmdInputChunks; -pub use mtmd_bitmap::MtmdBitmap; -pub use mtmd_bitmap_error::MtmdBitmapError; -pub use mtmd_capability::MtmdCapability; -pub use mtmd_context::MtmdContext; -pub use mtmd_context_params::MtmdContextParams; -pub use mtmd_context_params_error::MtmdContextParamsError; -pub use mtmd_default_marker::mtmd_default_marker; -pub use mtmd_default_marker_error::MtmdDefaultMarkerError; -pub use mtmd_encode_error::MtmdEncodeError; -pub use mtmd_eval_error::MtmdEvalError; -pub use mtmd_eval_limits::MtmdEvalLimits; -pub use mtmd_image_token_limit::MtmdImageTokenLimit; -pub use mtmd_init_error::MtmdInitError; -pub use mtmd_input_chunk::MtmdInputChunk; -pub use mtmd_input_chunk_error::MtmdInputChunkError; -pub use mtmd_input_chunk_type::MtmdInputChunkType; -pub use mtmd_input_chunk_type_error::MtmdInputChunkTypeError; -pub use mtmd_input_chunks::MtmdInputChunks; -pub use mtmd_input_chunks_error::MtmdInputChunksError; -pub use mtmd_input_part::MtmdInputPart; -pub use mtmd_input_text::MtmdInputText; -pub use mtmd_input_text_part::MtmdInputTextPart; -pub use mtmd_memory_usage::MtmdMemoryUsage; -pub use mtmd_memory_usage_error::MtmdMemoryUsageError; -pub use mtmd_tokenize_error::MtmdTokenizeError; -pub use non_causal_chunk_micro_batch_mismatch::NonCausalChunkMicroBatchMismatch; diff --git a/llama-cpp-bindings/src/mtmd/eval_chunk.rs b/llama-cpp-bindings/src/mtmd/eval_chunk.rs index 83e1bd244..a797164a5 100644 --- a/llama-cpp-bindings/src/mtmd/eval_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/eval_chunk.rs @@ -39,34 +39,34 @@ fn eval_chunk_single_status_to_result( Err(MtmdEvalError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_MTMD_CTX_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_eval_chunk_single", detail: "was given a null mtmd_ctx argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_LLAMA_CTX_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_eval_chunk_single", detail: "was given a null llama_ctx argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_CHUNK_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_eval_chunk_single", detail: "was given a null chunk argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_OUT_NEW_N_PAST_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_eval_chunk_single", detail: "was given a null out_new_n_past argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_mtmd_eval_chunk_single", code: i64::from(other), } @@ -196,7 +196,7 @@ mod eval_chunk_single_status_to_result_tests { assert_eq!( result, - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_eval_chunk_single", detail: "reported a thrown C++ exception without an error message", } @@ -210,10 +210,12 @@ mod eval_chunk_single_status_to_result_tests { assert_eq!( result, - Err(MtmdEvalError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_mtmd_eval_chunk_single", - code: 255, - })) + Err(MtmdEvalError::FfiStatus( + llama_cpp_ffi_status::FfiStatusError { + operation: "llama_rs_mtmd_eval_chunk_single", + code: 255, + } + )) ); } @@ -228,7 +230,7 @@ mod eval_chunk_single_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_eval_chunk_single", detail: "was given a null mtmd_ctx argument", } @@ -248,7 +250,7 @@ mod eval_chunk_single_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_eval_chunk_single", detail: "was given a null llama_ctx argument", } @@ -268,7 +270,7 @@ mod eval_chunk_single_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_eval_chunk_single", detail: "was given a null chunk argument", } @@ -288,7 +290,7 @@ mod eval_chunk_single_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_eval_chunk_single", detail: "was given a null out_new_n_past argument", } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs index 9c86c9e54..47b1e6bd1 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs @@ -35,7 +35,7 @@ unsafe fn from_file_status_to_result( match status { llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_OK => { let bitmap = NonNull::new(out_bitmap).ok_or_else(|| { - MtmdBitmapError::from(crate::FfiContractError { + MtmdBitmapError::from(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "success status contained a null bitmap", }) @@ -57,22 +57,22 @@ unsafe fn from_file_status_to_result( let message = unsafe { read_and_free_cpp_string(out_error, "llama_rs_mtmd_bitmap_init_from_file", "reported a thrown C++ exception without an error message") }?; Err(MtmdBitmapError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_CTX_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_CTX_ARG => Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "was given a null ctx argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_FNAME_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_FNAME_ARG => Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "was given a null fname argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_OUT_BITMAP_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_OUT_BITMAP_ARG => Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "was given a null out_bitmap argument", } .into()), - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_mtmd_bitmap_init_from_file", code: i64::from(other), } @@ -94,7 +94,7 @@ unsafe fn from_buf_status_to_result( match status { llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_BUF_OK => { let bitmap = NonNull::new(out_bitmap).ok_or_else(|| { - MtmdBitmapError::from(crate::FfiContractError { + MtmdBitmapError::from(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail: "success status contained a null bitmap", }) @@ -114,22 +114,22 @@ unsafe fn from_buf_status_to_result( let message = unsafe { read_and_free_cpp_string(out_error, "llama_rs_mtmd_bitmap_init_from_buf", "reported a thrown C++ exception without an error message") }?; Err(MtmdBitmapError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_BUF_NULL_CTX_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_BUF_NULL_CTX_ARG => Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail: "was given a null ctx argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_BUF_NULL_BUF_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_BUF_NULL_BUF_ARG => Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail: "was given a null buf argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_BUF_NULL_OUT_BITMAP_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_BUF_NULL_OUT_BITMAP_ARG => Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail: "was given a null out_bitmap argument", } .into()), - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_mtmd_bitmap_init_from_buf", code: i64::from(other), } @@ -148,21 +148,21 @@ fn bitmap_init_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_OK => NonNull::new(out_bitmap) .map(|bitmap| MtmdBitmap { bitmap }) .ok_or_else(|| { - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: OPERATION, detail: "success status contained a null bitmap", } .into() }), llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_NULL_DATA_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: OPERATION, detail: "was given a null data argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_NULL_OUT_BITMAP_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: OPERATION, detail: "was given a null out_bitmap argument", } @@ -185,7 +185,7 @@ fn bitmap_init_status_to_result( Err(MtmdBitmapError::Reported { message }) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: OPERATION, code: i64::from(other), } @@ -204,18 +204,18 @@ fn bitmap_init_from_audio_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_OK => NonNull::new(out_bitmap) .map(|bitmap| MtmdBitmap { bitmap }) .ok_or_else(|| { - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: OPERATION, detail: "success status contained a null bitmap", } .into() }), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_DATA_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_DATA_ARG => Err(llama_cpp_ffi_status::FfiContractError { operation: OPERATION, detail: "was given a null data argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_OUT_BITMAP_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_OUT_BITMAP_ARG => Err(llama_cpp_ffi_status::FfiContractError { operation: OPERATION, detail: "was given a null out_bitmap argument", } @@ -237,7 +237,7 @@ fn bitmap_init_from_audio_status_to_result( Err(MtmdBitmapError::Reported { message }) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: OPERATION, code: i64::from(other), } @@ -250,7 +250,7 @@ fn bitmap_init_from_audio_status_to_result( /// ```compile_fail,E0277 /// fn require_clone() {} /// -/// require_clone::(); +/// require_clone::(); /// ``` #[derive(Debug)] pub struct MtmdBitmap { @@ -408,7 +408,7 @@ impl Drop for MtmdBitmap { #[cfg(test)] mod tests { - use super::MtmdBitmap; + use crate::mtmd::mtmd_bitmap::MtmdBitmap; use crate::mtmd::mtmd_bitmap_error::MtmdBitmapError; #[test] @@ -548,8 +548,8 @@ mod from_file_status_to_result_tests { use std::ptr; use super::from_file_status_to_result; - use crate::FfiStatusError; use crate::mtmd::mtmd_bitmap_error::MtmdBitmapError; + use llama_cpp_ffi_status::FfiStatusError; #[test] fn from_file_success_with_null_bitmap_is_contract_error() { @@ -564,7 +564,7 @@ mod from_file_status_to_result_tests { assert_eq!( result.unwrap_err(), - MtmdBitmapError::FfiContract(crate::FfiContractError { + MtmdBitmapError::FfiContract(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "success status contained a null bitmap", }) @@ -617,7 +617,7 @@ mod from_file_status_to_result_tests { assert_eq!( result.unwrap_err(), - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "reported a thrown C++ exception without an error message", } @@ -639,7 +639,7 @@ mod from_file_status_to_result_tests { assert_eq!( result.unwrap_err(), - MtmdBitmapError::FfiContract(crate::FfiContractError { + MtmdBitmapError::FfiContract(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "was given a null ctx argument", }) @@ -659,7 +659,7 @@ mod from_file_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "was given a null fname argument", } @@ -681,7 +681,7 @@ mod from_file_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "was given a null out_bitmap argument", } @@ -785,7 +785,7 @@ mod from_buf_status_to_result_tests { assert_eq!( result.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail: "success status contained a null bitmap", } @@ -846,7 +846,7 @@ mod from_buf_status_to_result_tests { assert_eq!( result.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail: "reported a thrown C++ exception without an error message", } @@ -905,7 +905,7 @@ mod from_buf_status_to_result_tests { assert_eq!( result.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail, } @@ -925,7 +925,7 @@ mod from_buf_status_to_result_tests { assert_eq!( result.err(), Some( - crate::FfiStatusError { + llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_mtmd_bitmap_init_from_buf", code: i64::from(unrecognized), } @@ -944,7 +944,7 @@ mod bitmap_init_status_to_result_tests { use crate::mtmd::mtmd_bitmap_error::MtmdBitmapError; fn contract_violation(operation: &'static str, detail: &'static str) -> MtmdBitmapError { - crate::FfiContractError { operation, detail }.into() + llama_cpp_ffi_status::FfiContractError { operation, detail }.into() } #[test] @@ -983,10 +983,12 @@ mod bitmap_init_status_to_result_tests { operation, "reported a thrown C++ exception without an error message" )), - Some(MtmdBitmapError::FfiStatus(crate::FfiStatusError { - operation, - code: 255, - })), + Some(MtmdBitmapError::FfiStatus( + llama_cpp_ffi_status::FfiStatusError { + operation, + code: 255, + } + )), ] ); } @@ -1028,10 +1030,12 @@ mod bitmap_init_status_to_result_tests { operation, "reported a thrown C++ exception without an error message" )), - Some(MtmdBitmapError::FfiStatus(crate::FfiStatusError { - operation, - code: 255, - })), + Some(MtmdBitmapError::FfiStatus( + llama_cpp_ffi_status::FfiStatusError { + operation, + code: 255, + } + )), ] ); } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_bitmap_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_bitmap_error.rs index 606085f11..dcb70d98e 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_bitmap_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_bitmap_error.rs @@ -3,9 +3,9 @@ use std::path::PathBuf; #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdBitmapError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("Failed to create CString from bitmap-source path: {0}")] CStringError(#[from] std::ffi::NulError), #[error("Bitmap-source path is not valid UTF-8: {0:?}")] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_context.rs b/llama-cpp-bindings/src/mtmd/mtmd_context.rs index 5b15faeee..0d9fcbc23 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_context.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_context.rs @@ -81,28 +81,28 @@ fn map_tokenize_status( Err(MtmdTokenizeError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_BITMAPS_ARG_WHEN_NUM_BITMAPS_NONZERO => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "nonzero bitmap count was observed with a null bitmap array", } .into()) } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_CTX_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_CTX_ARG => Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "was given a null ctx argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_OUTPUT_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_OUTPUT_ARG => Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "was given a null output argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_TEXT_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_TEXT_ARG => Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "was given a null text argument", } .into()), - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_mtmd_tokenize", code: i64::from(other), } @@ -118,7 +118,7 @@ fn map_tokenize_from_parts_status( match status { llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_OK => Ok(()), llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_LLAMA_CPP_REPORTED_MALFORMED_PART => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "llama.cpp reported a malformed part list: a part holding both or neither of text and bitmap, a text part with a null text pointer, or bitmaps that do not line up with their parts", } @@ -149,34 +149,34 @@ fn map_tokenize_from_parts_status( Err(MtmdTokenizeError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_NULL_CTX_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "was given a null ctx argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_NULL_OUTPUT_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "was given a null output argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_NULL_PARTS_ARG_WHEN_NUM_PARTS_NONZERO => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "nonzero part count was observed with a null part array", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_NULL_PART_ELEMENT => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "was given a null element in the parts argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_mtmd_tokenize_from_parts", code: i64::from(other), } @@ -213,20 +213,20 @@ fn map_encode_chunk_status( Err(MtmdEncodeError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_ENCODE_CHUNK_NULL_CTX_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_encode_chunk", detail: "was given a null ctx argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_ENCODE_CHUNK_NULL_CHUNK_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_encode_chunk", detail: "was given a null chunk argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_mtmd_encode_chunk", code: i64::from(other), } @@ -243,7 +243,7 @@ fn map_init_from_file_status( match status { llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_OK => { let context = NonNull::new(out_ctx).ok_or_else(|| { - MtmdInitError::from(crate::FfiContractError { + MtmdInitError::from(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "success status contained a null multimodal context", }) @@ -272,27 +272,27 @@ fn map_init_from_file_status( Err(MtmdInitError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_NULL_MMPROJ_PATH_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "was given a null mmproj_path argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_NULL_TEXT_MODEL_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "was given a null text_model argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_NULL_OUT_CTX_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "was given a null out_ctx argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_mtmd_init_from_file", code: i64::from(other), } @@ -577,7 +577,7 @@ mod map_tokenize_from_parts_status_tests { ) .err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "llama.cpp reported a malformed part list: a part holding both or neither of text and bitmap, a text part with a null text pointer, or bitmaps that do not line up with their parts", } @@ -596,7 +596,7 @@ mod map_tokenize_from_parts_status_tests { ) .err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "was given a null element in the parts argument", } @@ -615,7 +615,7 @@ mod map_tokenize_from_parts_status_tests { ) .err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "reported a thrown C++ exception without an error message", } @@ -634,7 +634,7 @@ mod map_tokenize_from_parts_status_tests { ) .err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "was given a null ctx argument", } @@ -653,7 +653,7 @@ mod map_tokenize_from_parts_status_tests { ) .err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "was given a null output argument", } @@ -672,7 +672,7 @@ mod map_tokenize_from_parts_status_tests { ) .err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "nonzero part count was observed with a null part array", } @@ -681,10 +681,12 @@ mod map_tokenize_from_parts_status_tests { ); assert_eq!( map_tokenize_from_parts_status(255, 0, ptr::null_mut()).err(), - Some(MtmdTokenizeError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_mtmd_tokenize_from_parts", - code: 255, - })) + Some(MtmdTokenizeError::FfiStatus( + llama_cpp_ffi_status::FfiStatusError { + operation: "llama_rs_mtmd_tokenize_from_parts", + code: 255, + } + )) ); } } @@ -767,7 +769,7 @@ mod map_tokenize_status_tests { assert_eq!( result, - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "reported a thrown C++ exception without an error message", } @@ -785,10 +787,12 @@ mod map_tokenize_status_tests { assert_eq!( result, - Err(MtmdTokenizeError::FfiContract(crate::FfiContractError { - operation: "llama_rs_mtmd_tokenize", - detail: "nonzero bitmap count was observed with a null bitmap array", - })) + Err(MtmdTokenizeError::FfiContract( + llama_cpp_ffi_status::FfiContractError { + operation: "llama_rs_mtmd_tokenize", + detail: "nonzero bitmap count was observed with a null bitmap array", + } + )) ); } @@ -798,10 +802,12 @@ mod map_tokenize_status_tests { assert_eq!( result, - Err(MtmdTokenizeError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_mtmd_tokenize", - code: 255, - })) + Err(MtmdTokenizeError::FfiStatus( + llama_cpp_ffi_status::FfiStatusError { + operation: "llama_rs_mtmd_tokenize", + code: 255, + } + )) ); } @@ -815,7 +821,7 @@ mod map_tokenize_status_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "was given a null ctx argument", } @@ -834,7 +840,7 @@ mod map_tokenize_status_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "was given a null output argument", } @@ -853,7 +859,7 @@ mod map_tokenize_status_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "was given a null text argument", } @@ -940,7 +946,7 @@ mod map_encode_chunk_status_tests { assert_eq!( result, - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_encode_chunk", detail: "reported a thrown C++ exception without an error message", } @@ -954,10 +960,12 @@ mod map_encode_chunk_status_tests { assert_eq!( result, - Err(MtmdEncodeError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_mtmd_encode_chunk", - code: 255, - })) + Err(MtmdEncodeError::FfiStatus( + llama_cpp_ffi_status::FfiStatusError { + operation: "llama_rs_mtmd_encode_chunk", + code: 255, + } + )) ); } @@ -971,7 +979,7 @@ mod map_encode_chunk_status_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_encode_chunk", detail: "was given a null ctx argument", } @@ -990,7 +998,7 @@ mod map_encode_chunk_status_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_encode_chunk", detail: "was given a null chunk argument", } @@ -1045,7 +1053,7 @@ mod map_init_from_file_status_tests { assert_eq!( result.unwrap_err(), - MtmdInitError::FfiContract(crate::FfiContractError { + MtmdInitError::FfiContract(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "success status contained a null multimodal context", }) @@ -1075,7 +1083,7 @@ mod map_init_from_file_status_tests { assert_eq!( result.unwrap_err(), - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "reported a thrown C++ exception without an error message", } @@ -1090,7 +1098,7 @@ mod map_init_from_file_status_tests { assert_eq!( result.unwrap_err(), - MtmdInitError::FfiStatus(crate::FfiStatusError { + MtmdInitError::FfiStatus(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_mtmd_init_from_file", code: 255, }) @@ -1108,7 +1116,7 @@ mod map_init_from_file_status_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "was given a null mmproj_path argument", } @@ -1128,7 +1136,7 @@ mod map_init_from_file_status_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "was given a null text_model argument", } @@ -1148,7 +1156,7 @@ mod map_init_from_file_status_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "was given a null out_ctx argument", } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_context_params.rs b/llama-cpp-bindings/src/mtmd/mtmd_context_params.rs index ebf5e630a..d7898fea9 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_context_params.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_context_params.rs @@ -71,7 +71,7 @@ impl From for MtmdContextParams { mod tests { use std::num::NonZeroU32; - use super::MtmdContextParams; + use crate::mtmd::mtmd_context_params::MtmdContextParams; use crate::mtmd::mtmd_context_params_error::MtmdContextParamsError; use crate::mtmd::mtmd_image_token_limit::MtmdImageTokenLimit; diff --git a/llama-cpp-bindings/src/mtmd/mtmd_default_marker.rs b/llama-cpp-bindings/src/mtmd/mtmd_default_marker.rs index 33e7186a1..553c91622 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_default_marker.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_default_marker.rs @@ -25,7 +25,7 @@ mod tests { use std::str; use super::marker_bytes_to_str; - use super::mtmd_default_marker; + use crate::mtmd::mtmd_default_marker::mtmd_default_marker; use crate::mtmd::mtmd_default_marker_error::MtmdDefaultMarkerError; #[test] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_encode_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_encode_error.rs index 0906829ab..7f08ccaf7 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_encode_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_encode_error.rs @@ -1,9 +1,9 @@ #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdEncodeError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("multimodal chunk encoding failed with code: {code}")] EncodingFailed { code: i32 }, #[error("not enough memory")] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs index b6824a2b3..d77f03e35 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs @@ -5,9 +5,9 @@ use crate::mtmd::non_causal_chunk_micro_batch_mismatch::NonCausalChunkMicroBatch #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdEvalError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("batch size {requested} exceeds context batch size {context_max}")] BatchSizeExceedsContextLimit { requested: i32, context_max: u32 }, #[error("batch size {requested} must be positive")] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_eval_limits.rs b/llama-cpp-bindings/src/mtmd/mtmd_eval_limits.rs index 340deb384..d7dddfb9a 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_eval_limits.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_eval_limits.rs @@ -53,10 +53,10 @@ impl MtmdEvalLimits { mod tests { use std::num::NonZeroU32; - use super::MtmdEvalLimits; use crate::context::context_limits::ContextLimits; use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; use crate::mtmd::mtmd_eval_error::MtmdEvalError; + use crate::mtmd::mtmd_eval_limits::MtmdEvalLimits; const LIMITS: MtmdEvalLimits = MtmdEvalLimits { context: ContextLimits { diff --git a/llama-cpp-bindings/src/mtmd/mtmd_image_token_limit.rs b/llama-cpp-bindings/src/mtmd/mtmd_image_token_limit.rs index c2a832ba7..9f945c36d 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_image_token_limit.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_image_token_limit.rs @@ -42,7 +42,7 @@ impl MtmdImageTokenLimit { mod tests { use std::num::NonZeroU32; - use super::MtmdImageTokenLimit; + use crate::mtmd::mtmd_image_token_limit::MtmdImageTokenLimit; const SIXTY_FOUR: NonZeroU32 = NonZeroU32::new(64).unwrap(); diff --git a/llama-cpp-bindings/src/mtmd/mtmd_init_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_init_error.rs index 21ea9deaa..03190edb5 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_init_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_init_error.rs @@ -5,9 +5,9 @@ use super::mtmd_context_params_error::MtmdContextParamsError; #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdInitError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error(transparent)] ContextParams(#[from] MtmdContextParamsError), #[error("Failed to create CString from mmproj path: {0}")] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs index 3d90e885a..ed54c311d 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs @@ -65,21 +65,21 @@ fn input_chunk_copy_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_OK => NonNull::new(out_chunk) .map(|chunk| MtmdInputChunk { chunk, owned: true }) .ok_or_else(|| { - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: OPERATION, detail: "success status contained a null chunk", } .into() }), llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_NULL_CHUNK_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: OPERATION, detail: "was given a null chunk argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_NULL_OUT_CHUNK_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: OPERATION, detail: "was given a null out_chunk argument", } @@ -102,7 +102,7 @@ fn input_chunk_copy_status_to_result( Err(MtmdInputChunkError::Reported { message }) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: OPERATION, code: i64::from(other), } @@ -279,7 +279,7 @@ mod input_chunk_copy_status_to_result_tests { const OPERATION: &str = "llama_rs_mtmd_input_chunk_copy"; fn contract_violation(detail: &'static str) -> MtmdInputChunkError { - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: OPERATION, detail, } @@ -311,10 +311,12 @@ mod input_chunk_copy_status_to_result_tests { Some(contract_violation( "reported a thrown C++ exception without an error message" )), - Some(MtmdInputChunkError::FfiStatus(crate::FfiStatusError { - operation: OPERATION, - code: 255, - })), + Some(MtmdInputChunkError::FfiStatus( + llama_cpp_ffi_status::FfiStatusError { + operation: OPERATION, + code: 255, + } + )), ] ); } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_error.rs index 5a72a80a0..b1607a042 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_error.rs @@ -1,9 +1,9 @@ #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdInputChunkError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_type.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_type.rs index 5392d85e0..5d2473187 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_type.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_type.rs @@ -25,7 +25,7 @@ impl TryFrom for MtmdInputChunkTy #[cfg(test)] mod tests { - use super::MtmdInputChunkType; + use crate::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; use crate::mtmd::mtmd_input_chunk_type_error::MtmdInputChunkTypeError; #[test] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs index cba0be724..7d9f85cae 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs @@ -25,14 +25,14 @@ fn input_chunks_init_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_OK => NonNull::new(out_chunks) .map(|chunks| MtmdInputChunks { chunks }) .ok_or_else(|| { - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: OPERATION, detail: "success status contained null chunks", } .into() }), llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_NULL_OUT_CHUNKS_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: OPERATION, detail: "was given a null out_chunks argument", } @@ -55,7 +55,7 @@ fn input_chunks_init_status_to_result( Err(MtmdInputChunksError::Reported { message }) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: OPERATION, code: i64::from(other), } @@ -187,7 +187,7 @@ impl MtmdInputChunks { fn chunk_within_count(&self, index: usize) -> Result { self.get(index).ok_or_else(|| { - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "mtmd_input_chunks_get", detail: "returned a null chunk within the chunk count", } @@ -204,7 +204,7 @@ impl Drop for MtmdInputChunks { #[cfg(test)] mod tests { - use super::MtmdInputChunks; + use crate::mtmd::mtmd_input_chunks::MtmdInputChunks; #[test] fn new_creates_empty_chunks() { @@ -233,7 +233,7 @@ mod input_chunks_init_status_to_result_tests { const OPERATION: &str = "llama_rs_mtmd_input_chunks_init"; fn contract_violation(detail: &'static str) -> MtmdInputChunksError { - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: OPERATION, detail, } @@ -263,10 +263,12 @@ mod input_chunks_init_status_to_result_tests { Some(contract_violation( "reported a thrown C++ exception without an error message" )), - Some(MtmdInputChunksError::FfiStatus(crate::FfiStatusError { - operation: OPERATION, - code: 255, - })), + Some(MtmdInputChunksError::FfiStatus( + llama_cpp_ffi_status::FfiStatusError { + operation: OPERATION, + code: 255, + } + )), ] ); } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks_error.rs index 195a5c62d..6b2e68385 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks_error.rs @@ -1,9 +1,9 @@ #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdInputChunksError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_part.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_part.rs index 178d23ed7..31a43cadd 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_part.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_part.rs @@ -11,7 +11,7 @@ pub enum MtmdInputPart<'bitmap> { mod tests { use std::thread; - use super::MtmdInputPart; + use crate::mtmd::mtmd_input_part::MtmdInputPart; use crate::mtmd::mtmd_input_text_part::MtmdInputTextPart; #[test] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs index c34fa3131..703b1d72d 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs @@ -60,7 +60,7 @@ fn get_memory_usage_status_to_result( Err(MtmdMemoryUsageError::Reported { message }) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: OPERATION, code: i64::from(other), } @@ -69,7 +69,7 @@ fn get_memory_usage_status_to_result( } const fn contract_violation(detail: &'static str) -> MtmdMemoryUsageError { - MtmdMemoryUsageError::FfiContract(crate::FfiContractError { + MtmdMemoryUsageError::FfiContract(llama_cpp_ffi_status::FfiContractError { operation: OPERATION, detail, }) @@ -183,11 +183,11 @@ mod tests { use llama_cpp_fault_injection::ArmedFault; use llama_cpp_fault_injection::injected_fault_message; - use super::MtmdMemoryUsage; use super::get_memory_usage_status_to_result; use crate::mtmd::mtmd_context_params::MtmdContextParams; use crate::mtmd::mtmd_context_params_error::MtmdContextParamsError; use crate::mtmd::mtmd_image_token_limit::MtmdImageTokenLimit; + use crate::mtmd::mtmd_memory_usage::MtmdMemoryUsage; use crate::mtmd::mtmd_memory_usage_error::MtmdMemoryUsageError; const MICRO_BATCH_TOKENS: NonZeroU32 = NonZeroU32::new(64).unwrap(); @@ -219,7 +219,7 @@ mod tests { } fn contract_violation(detail: &'static str) -> MtmdMemoryUsageError { - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_mtmd_get_memory_usage", detail, } @@ -427,10 +427,12 @@ mod tests { fn an_unknown_status_is_preserved() { assert_eq!( get_memory_usage_status_to_result(255, ptr::null_mut()), - Err(MtmdMemoryUsageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_mtmd_get_memory_usage", - code: 255, - })) + Err(MtmdMemoryUsageError::FfiStatus( + llama_cpp_ffi_status::FfiStatusError { + operation: "llama_rs_mtmd_get_memory_usage", + code: 255, + } + )) ); } } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage_error.rs index ac339cafc..0d9417d19 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage_error.rs @@ -5,9 +5,9 @@ use super::mtmd_context_params_error::MtmdContextParamsError; #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdMemoryUsageError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error(transparent)] ContextParams(#[from] MtmdContextParamsError), #[error("the memory usage of mmproj {path:?} could not be measured, because it does not load")] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_tokenize_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_tokenize_error.rs index 29c36b004..d50f828cc 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_tokenize_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_tokenize_error.rs @@ -3,9 +3,9 @@ use crate::mtmd::mtmd_input_chunks_error::MtmdInputChunksError; #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdTokenizeError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), #[error("Failed to create CString from input text: {0}")] CStringError(#[from] std::ffi::NulError), #[error("{0}")] diff --git a/llama-cpp-bindings/src/sampled_token.rs b/llama-cpp-bindings/src/sampled_token.rs index 94f13957c..c3a6613c5 100644 --- a/llama-cpp-bindings/src/sampled_token.rs +++ b/llama-cpp-bindings/src/sampled_token.rs @@ -43,7 +43,7 @@ impl SampledToken { #[cfg(test)] mod tests { - use super::SampledToken; + use crate::sampled_token::SampledToken; use crate::sampled_token_section::SampledTokenSection; use crate::token::LlamaToken; diff --git a/llama-cpp-bindings/src/sampled_token_classifier.rs b/llama-cpp-bindings/src/sampled_token_classifier.rs index f47b2460d..f86261d05 100644 --- a/llama-cpp-bindings/src/sampled_token_classifier.rs +++ b/llama-cpp-bindings/src/sampled_token_classifier.rs @@ -11,29 +11,28 @@ use crate::chat_output::ChatOutput; use crate::chat_output_completeness::ChatOutputCompleteness; use crate::chat_piece_role::ChatPieceRole; use crate::chat_session::ChatSession; +use crate::classified_sample::ClassifiedSample; use crate::context::LlamaContext; -use crate::error::EvalMultimodalChunksError; -use crate::error::SampleError; +use crate::error::eval_multimodal_chunks_error::EvalMultimodalChunksError; use crate::error::ingest_error::IngestError; +use crate::error::sample_error::SampleError; use crate::eval_multimodal_chunks_params::EvalMultimodalChunksParams; use crate::generation_progress::GenerationProgress; use crate::ingest_outcome::IngestOutcome; use crate::llama_batch::LlamaBatch; use crate::model::LlamaModel; -use crate::mtmd::FittedMtmdInputChunk; -use crate::mtmd::FittedMtmdInputChunks; -use crate::mtmd::MtmdContext; -use crate::mtmd::MtmdInputChunkType; -use crate::mtmd::MtmdInputChunks; +use crate::mtmd::fitted_mtmd_input_chunk::FittedMtmdInputChunk; +use crate::mtmd::fitted_mtmd_input_chunks::FittedMtmdInputChunks; +use crate::mtmd::mtmd_context::MtmdContext; use crate::mtmd::mtmd_eval_limits::MtmdEvalLimits; +use crate::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; +use crate::mtmd::mtmd_input_chunks::MtmdInputChunks; use crate::sampled_token::SampledToken; +use crate::sampled_token_section::SampledTokenSection; use crate::sampling::LlamaSampler; use crate::token::LlamaToken; use crate::token_piece::TokenPiece; -pub use crate::classified_sample::ClassifiedSample; -pub use crate::sampled_token_section::SampledTokenSection; - enum ContentProbe { Disabled, BareJson(BareJsonToolCallProbe), diff --git a/llama-cpp-bindings/src/sampling.rs b/llama-cpp-bindings/src/sampling.rs index 3544469cd..b6fa18ff8 100644 --- a/llama-cpp-bindings/src/sampling.rs +++ b/llama-cpp-bindings/src/sampling.rs @@ -8,13 +8,16 @@ use llama_cpp_error_recorder::RecordedError; use crate::context::LlamaContext; use crate::dry_sequence_breakers::dry_sequence_breakers; -use crate::error::SamplerApplyError; +use crate::error::grammar_error::GrammarError; +use crate::error::sample_error::SampleError; +use crate::error::sampler_accept_error::SamplerAcceptError; +use crate::error::sampler_apply_error::SamplerApplyError; +use crate::error::sampling_error::SamplingError; use crate::model::LlamaModel; use crate::sanitized_grammar::SanitizedGrammar; use crate::token::LlamaToken; use crate::token::data_array::LlamaTokenDataArray; use crate::token::logit_bias::LlamaLogitBias; -use crate::{GrammarError, SampleError, SamplerAcceptError, SamplingError}; use llama_cpp_ffi_status::read_and_free_cpp_string; use llama_cpp_gbnf::gbnf_validation_error::GbnfValidationError; use llama_cpp_gbnf::validate_gbnf::validate_gbnf; @@ -42,20 +45,20 @@ fn check_sampler_accept_status( Err(SamplerAcceptError::GrammarStateCorrupted { message }) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_NULL_SAMPLER_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_accept", detail: "was given a null sampler argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_accept", detail: "was given a null out_error argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_sampler_accept", code: i64::from(other), } @@ -89,20 +92,20 @@ fn sampler_apply_status_to_result( Err(SamplerApplyError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_DATA_ARRAY_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_apply", detail: "was given a null data_array argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_apply", detail: "was given a null out_error argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_sampler_apply", code: i64::from(other), } @@ -134,34 +137,34 @@ fn sampler_sample_status_to_result( Err(SampleError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_SAMPLER_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_sample", detail: "was given a null sampler argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_CTX_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_sample", detail: "was given a null ctx argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_OUT_TOKEN_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_sample", detail: "was given a null out_token argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_sample", detail: "was given a null out_error argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_sampler_sample", code: i64::from(other), } @@ -198,20 +201,20 @@ fn sampler_init_grammar_status_to_result( Err(GrammarError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_NULL_OUT_SAMPLER_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_init_grammar", detail: "was given a null out_sampler argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_init_grammar", detail: "was given a null out_error argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_sampler_init_grammar", code: i64::from(other), } @@ -245,17 +248,17 @@ fn sampler_init_grammar_lazy_patterns_status_to_result( let message = unsafe { read_and_free_cpp_string(error_ptr, "llama_rs_sampler_init_grammar_lazy_patterns", "reported a thrown C++ exception without an error message") }?; Err(GrammarError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_NULL_OUT_SAMPLER_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_NULL_OUT_SAMPLER_ARG => Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", detail: "was given a null out_sampler argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_NULL_OUT_ERROR_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_NULL_OUT_ERROR_ARG => Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", detail: "was given a null out_error argument", } .into()), - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", code: i64::from(other), } @@ -300,20 +303,20 @@ fn sampler_init_status_to_result( }) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_NULL_OUT_SAMPLER_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_init", detail: "was given a null out_sampler argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_init", detail: "was given a null out_error argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_sampler_init", code: i64::from(other), } @@ -347,27 +350,27 @@ fn sampler_chain_add_status_to_result( }) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_NULL_CHAIN_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "was given a null chain argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_NULL_SAMPLER_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "was given a null sampler argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "was given a null out_error argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_sampler_chain_add", code: i64::from(other), } @@ -974,8 +977,8 @@ mod tests { use llama_cpp_error_recorder::record; use super::LlamaSampler; - use crate::GrammarError; - use crate::SampleError; + use crate::error::grammar_error::GrammarError; + use crate::error::sample_error::SampleError; use crate::sanitized_grammar::SanitizedGrammar; unsafe extern "C" fn record_a_failure_and_exclude_every_candidate( @@ -1057,7 +1060,7 @@ mod tests { assert_eq!( result.unwrap_err(), - crate::SamplingError::SamplerUnavailable { sampler: "test" } + crate::error::sampling_error::SamplingError::SamplerUnavailable { sampler: "test" } ); } @@ -1265,7 +1268,7 @@ mod checked_usize_as_i32_sampling_tests { use super::LlamaSampler; use super::checked_usize_as_i32_sampling; - use crate::error::SamplingError; + use crate::error::sampling_error::SamplingError; use crate::token::logit_bias::LlamaLogitBias; #[test] @@ -1296,7 +1299,7 @@ mod grammar_callback_error_to_result_tests { use llama_cpp_error_recorder::RecordedError; use super::grammar_callback_error_to_result; - use crate::SampleError; + use crate::error::sample_error::SampleError; #[test] fn grammar_callback_error_to_result_maps_recorded_error() { @@ -1322,7 +1325,7 @@ mod grammar_callback_error_to_accept_result_tests { use llama_cpp_error_recorder::RecordedError; use super::grammar_callback_error_to_accept_result; - use crate::SamplerAcceptError; + use crate::error::sampler_accept_error::SamplerAcceptError; #[test] fn grammar_callback_error_to_accept_result_maps_recorded_error() { @@ -1349,9 +1352,9 @@ mod check_sampler_accept_status_tests { use std::ptr; use super::check_sampler_accept_status; - use crate::FfiContractError; - use crate::FfiStatusError; - use crate::SamplerAcceptError; + use crate::error::sampler_accept_error::SamplerAcceptError; + use llama_cpp_ffi_status::FfiContractError; + use llama_cpp_ffi_status::FfiStatusError; #[test] fn check_sampler_accept_status_ok() { @@ -1468,8 +1471,8 @@ mod sampler_sample_status_to_result_tests { use std::ptr; use super::sampler_sample_status_to_result; - use crate::FfiStatusError; - use crate::SampleError; + use crate::error::sample_error::SampleError; + use llama_cpp_ffi_status::FfiStatusError; #[test] fn sampler_sample_status_allocation_failure_maps_to_not_enough_memory() { @@ -1492,7 +1495,7 @@ mod sampler_sample_status_to_result_tests { assert_eq!( result.unwrap_err(), - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_sample", detail: "reported a thrown C++ exception without an error message", } @@ -1507,10 +1510,12 @@ mod sampler_sample_status_to_result_tests { assert_eq!( result, - Err(SampleError::FfiContract(crate::FfiContractError { - operation: "llama_rs_sampler_sample", - detail: "was given a null ctx argument", - })) + Err(SampleError::FfiContract( + llama_cpp_ffi_status::FfiContractError { + operation: "llama_rs_sampler_sample", + detail: "was given a null ctx argument", + } + )) ); } @@ -1524,7 +1529,7 @@ mod sampler_sample_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_sample", detail: "was given a null sampler argument", } @@ -1543,7 +1548,7 @@ mod sampler_sample_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_sample", detail: "was given a null out_token argument", } @@ -1562,7 +1567,7 @@ mod sampler_sample_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_sample", detail: "was given a null out_error argument", } @@ -1633,9 +1638,11 @@ mod sampler_init_status_to_result_tests { "penalties", ) .err(), - Some(crate::SamplingError::SamplerUnavailable { - sampler: "penalties" - }) + Some( + crate::error::sampling_error::SamplingError::SamplerUnavailable { + sampler: "penalties" + } + ) ); } @@ -1649,7 +1656,7 @@ mod sampler_init_status_to_result_tests { "penalties", ) .err(), - Some(crate::SamplingError::NotEnoughMemory) + Some(crate::error::sampling_error::SamplingError::NotEnoughMemory) ); } @@ -1663,9 +1670,11 @@ mod sampler_init_status_to_result_tests { "penalties", ) .err(), - Some(crate::SamplingError::LlamaCppOutOfMemory { - sampler: "penalties" - }) + Some( + crate::error::sampling_error::SamplingError::LlamaCppOutOfMemory { + sampler: "penalties" + } + ) ); } @@ -1682,7 +1691,7 @@ mod sampler_init_status_to_result_tests { "penalties", ) .err(), - Some(crate::SamplingError::Reported { + Some(crate::error::sampling_error::SamplingError::Reported { sampler: "penalties", message: "vector too long".to_owned(), }) @@ -1700,7 +1709,7 @@ mod sampler_init_status_to_result_tests { ) .err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_init", detail: "was given a null out_sampler argument", } @@ -1720,7 +1729,7 @@ mod sampler_init_status_to_result_tests { ) .err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_init", detail: "was given a null out_error argument", } @@ -1740,7 +1749,7 @@ mod sampler_init_status_to_result_tests { ) .err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_init", detail: "reported a thrown C++ exception without an error message", } @@ -1760,7 +1769,7 @@ mod sampler_init_status_to_result_tests { ) .err(), Some( - crate::FfiStatusError { + llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_sampler_init", code: i64::from( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION @@ -1786,7 +1795,7 @@ mod sampler_chain_add_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_ERROR_STRING_ALLOCATION_FAILED, ptr::null_mut(), ), - Err(crate::SamplingError::NotEnoughMemory) + Err(crate::error::sampling_error::SamplingError::NotEnoughMemory) ); } @@ -1797,7 +1806,11 @@ mod sampler_chain_add_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), ), - Err(crate::SamplingError::LlamaCppOutOfMemory { sampler: "chain" }) + Err( + crate::error::sampling_error::SamplingError::LlamaCppOutOfMemory { + sampler: "chain" + } + ) ); } @@ -1811,7 +1824,7 @@ mod sampler_chain_add_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION, out_error, ), - Err(crate::SamplingError::Reported { + Err(crate::error::sampling_error::SamplingError::Reported { sampler: "chain", message: "vector too long".to_owned(), }) @@ -1825,7 +1838,7 @@ mod sampler_chain_add_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_NULL_CHAIN_ARG, ptr::null_mut(), ), - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "was given a null chain argument", } @@ -1840,7 +1853,7 @@ mod sampler_chain_add_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_NULL_SAMPLER_ARG, ptr::null_mut(), ), - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "was given a null sampler argument", } @@ -1855,7 +1868,7 @@ mod sampler_chain_add_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_NULL_OUT_ERROR_ARG, ptr::null_mut(), ), - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "was given a null out_error argument", } @@ -1870,7 +1883,7 @@ mod sampler_chain_add_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), ), - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "reported a thrown C++ exception without an error message", } @@ -1886,7 +1899,7 @@ mod sampler_chain_add_status_to_result_tests { + 1, ptr::null_mut(), ), - Err(crate::FfiStatusError { + Err(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_sampler_chain_add", code: i64::from( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION @@ -1903,8 +1916,8 @@ mod sampler_init_grammar_status_to_result_tests { use std::ptr; use super::sampler_init_grammar_status_to_result; - use crate::FfiStatusError; - use crate::GrammarError; + use crate::error::grammar_error::GrammarError; + use llama_cpp_ffi_status::FfiStatusError; #[test] fn sampler_init_grammar_status_null_maps_to_grammar_malformed() { @@ -1938,7 +1951,7 @@ mod sampler_init_grammar_status_to_result_tests { assert_eq!( result.unwrap_err(), - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_init_grammar", detail: "reported a thrown C++ exception without an error message", } @@ -1954,7 +1967,7 @@ mod sampler_init_grammar_status_to_result_tests { assert_eq!( result.unwrap_err(), - GrammarError::FfiContract(crate::FfiContractError { + GrammarError::FfiContract(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_init_grammar", detail: "was given a null out_sampler argument", }) @@ -1971,7 +1984,7 @@ mod sampler_init_grammar_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_init_grammar", detail: "was given a null out_error argument", } @@ -2034,8 +2047,8 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { use std::ptr; use super::sampler_init_grammar_lazy_patterns_status_to_result; - use crate::FfiStatusError; - use crate::GrammarError; + use crate::error::grammar_error::GrammarError; + use llama_cpp_ffi_status::FfiStatusError; #[test] fn sampler_init_grammar_lazy_patterns_status_null_maps_to_lazy_patterns_grammar_malformed() { @@ -2069,7 +2082,7 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { assert_eq!( result.unwrap_err(), - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", detail: "reported a thrown C++ exception without an error message", } @@ -2087,7 +2100,7 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { ) .err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", detail: "reported a thrown C++ exception without an error message", } @@ -2107,7 +2120,7 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { assert_eq!( result.unwrap_err(), - GrammarError::FfiContract(crate::FfiContractError { + GrammarError::FfiContract(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", detail: "was given a null out_sampler argument", }) @@ -2124,7 +2137,7 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", detail: "was given a null out_error argument", } @@ -2184,9 +2197,9 @@ mod sampler_application_tests { use llama_cpp_fault_injection::injected_fault_message; use super::LlamaSampler; - use crate::SampleError; - use crate::SamplingError; - use crate::error::SamplerApplyError; + use crate::error::sample_error::SampleError; + use crate::error::sampler_apply_error::SamplerApplyError; + use crate::error::sampling_error::SamplingError; use crate::token::LlamaToken; use crate::token::data::LlamaTokenData; use crate::token::data_array::LlamaTokenDataArray; @@ -2240,7 +2253,7 @@ mod sampler_application_tests { assert_eq!( overreporting_sampler.apply(&mut excluded_candidates()), Err(SampleError::SamplerApply(SamplerApplyError::FfiContract( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "modify_as_c_llama_token_data_array", detail: "the llama.cpp sampler grew the token data array beyond its capacity", } @@ -2286,7 +2299,7 @@ mod sampler_apply_status_to_result_tests { use std::ptr; use super::sampler_apply_status_to_result; - use crate::error::SamplerApplyError; + use crate::error::sampler_apply_error::SamplerApplyError; #[test] fn sampler_apply_status_allocation_failed_returns_not_enough_memory() { @@ -2306,7 +2319,7 @@ mod sampler_apply_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), ), - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_apply", detail: "reported a thrown C++ exception without an error message", } @@ -2320,10 +2333,12 @@ mod sampler_apply_status_to_result_tests { assert_eq!( result, - Err(SamplerApplyError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_sampler_apply", - code: 255, - })) + Err(SamplerApplyError::FfiStatus( + llama_cpp_ffi_status::FfiStatusError { + operation: "llama_rs_sampler_apply", + code: 255, + } + )) ); } @@ -2345,7 +2360,7 @@ mod sampler_apply_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_apply", detail: "was given a null data_array argument", } @@ -2363,7 +2378,7 @@ mod sampler_apply_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + llama_cpp_ffi_status::FfiContractError { operation: "llama_rs_sampler_apply", detail: "was given a null out_error argument", } diff --git a/llama-cpp-bindings/src/send_logs_to_log.rs b/llama-cpp-bindings/src/send_logs_to_log.rs index f4e94b736..6a91b2e57 100644 --- a/llama-cpp-bindings/src/send_logs_to_log.rs +++ b/llama-cpp-bindings/src/send_logs_to_log.rs @@ -174,11 +174,15 @@ mod tests { use log::{Level, Log, Metadata, Record}; use serial_test::serial; - use super::{ - GGML_SOURCE, LLAMA_SOURCE, LogSource, dispatch_output, ggml_level_to_incoming, logs_to_log, - resolve_record, send_logs_to_log, - }; + use super::GGML_SOURCE; + use super::LLAMA_SOURCE; + use super::LogSource; + use super::dispatch_output; + use super::ggml_level_to_incoming; + use super::logs_to_log; + use super::resolve_record; use crate::log_options::LogOptions; + use crate::send_logs_to_log::send_logs_to_log; #[derive(Debug)] struct CapturedRecord { diff --git a/llama-cpp-bindings/src/token/data_array.rs b/llama-cpp-bindings/src/token/data_array.rs index d58adf1cc..c80753dab 100644 --- a/llama-cpp-bindings/src/token/data_array.rs +++ b/llama-cpp-bindings/src/token/data_array.rs @@ -48,7 +48,7 @@ impl LlamaTokenDataArray { impl LlamaTokenDataArray { /// # Errors /// - /// Returns [`crate::FfiContractError`] when the llama.cpp sampler grows the array beyond the + /// Returns [`llama_cpp_ffi_status::FfiContractError`] when the llama.cpp sampler grows the array beyond the /// capacity this buffer was allocated with, which would make the following `set_len` /// undefined behaviour. /// @@ -60,7 +60,7 @@ impl LlamaTokenDataArray { pub unsafe fn modify_as_c_llama_token_data_array( &mut self, modify: impl FnOnce(&mut llama_cpp_bindings_sys::llama_token_data_array) -> TResult, - ) -> Result { + ) -> Result { let size = self.data.len(); let data = self .data @@ -80,7 +80,7 @@ impl LlamaTokenDataArray { let result = modify(&mut c_llama_token_data_array); if c_llama_token_data_array.size > self.data.capacity() { - return Err(crate::FfiContractError { + return Err(llama_cpp_ffi_status::FfiContractError { operation: "modify_as_c_llama_token_data_array", detail: "the llama.cpp sampler grew the token data array beyond its capacity", }); @@ -226,7 +226,7 @@ mod tests { assert_eq!( result, - Err(crate::FfiContractError { + Err(llama_cpp_ffi_status::FfiContractError { operation: "modify_as_c_llama_token_data_array", detail: "the llama.cpp sampler grew the token data array beyond its capacity", }) diff --git a/llama-cpp-bindings/src/token_piece.rs b/llama-cpp-bindings/src/token_piece.rs index 5bef2b216..d7ca5fd1b 100644 --- a/llama-cpp-bindings/src/token_piece.rs +++ b/llama-cpp-bindings/src/token_piece.rs @@ -23,7 +23,7 @@ impl TokenPiece { #[cfg(test)] mod tests { - use super::TokenPiece; + use crate::token_piece::TokenPiece; #[test] fn a_marker_is_raw_text_without_a_visible_part() { diff --git a/llama-cpp-bindings/src/tool_call_format.rs b/llama-cpp-bindings/src/tool_call_format.rs index 131032a7a..9e67f55c3 100644 --- a/llama-cpp-bindings/src/tool_call_format.rs +++ b/llama-cpp-bindings/src/tool_call_format.rs @@ -9,12 +9,11 @@ pub mod separator_split; pub mod tool_call_format_outcome; pub mod xml_function_tags; -pub use self::tool_call_format_outcome::ToolCallFormatOutcome; - use llama_cpp_bindings_types::ToolCallArgsShape; use llama_cpp_bindings_types::ToolCallMarkers; -use crate::error::ToolCallFormatFailure; +use crate::error::tool_call_format_failure::ToolCallFormatFailure; +use crate::tool_call_format::tool_call_format_outcome::ToolCallFormatOutcome; #[must_use] pub fn try_parse(body: &str, markers: &ToolCallMarkers) -> ToolCallFormatOutcome { @@ -61,8 +60,8 @@ mod tests { use super::ToolCallFormatOutcome; use super::try_parse; - use crate::error::BracketedArgsFailure; - use crate::error::ToolCallFormatFailure; + use crate::error::bracketed_args_failure::BracketedArgsFailure; + use crate::error::tool_call_format_failure::ToolCallFormatFailure; fn mistral3_markers() -> ToolCallMarkers { ToolCallMarkers { @@ -235,7 +234,7 @@ mod tests { #[test] fn try_parse_returns_no_match_for_plain_content_under_every_known_shape() { - use crate::chat_template_tool_calls::known_marker_candidates; + use crate::chat_template_tool_calls::known_marker_candidates::known_marker_candidates; let plain_content = "Sorry, I cannot help with that request."; @@ -252,7 +251,7 @@ mod tests { fn duck_type_resolves_qwen_xml_input_via_xml_tags_shape_first() { use llama_cpp_bindings_types::ToolCallArguments; - use crate::chat_template_tool_calls::known_marker_candidates; + use crate::chat_template_tool_calls::known_marker_candidates::known_marker_candidates; let qwen_input = "\n\ \n\ @@ -295,7 +294,7 @@ mod tests { fn duck_type_resolves_glm_input_via_key_value_xml_tags_shape() { use llama_cpp_bindings_types::ToolCallArguments; - use crate::chat_template_tool_calls::known_marker_candidates; + use crate::chat_template_tool_calls::known_marker_candidates::known_marker_candidates; let glm_input = "get_weather\ location\ @@ -334,7 +333,7 @@ mod tests { fn duck_type_resolves_mistral_input_via_bracketed_json_shape() { use llama_cpp_bindings_types::ToolCallArguments; - use crate::chat_template_tool_calls::known_marker_candidates; + use crate::chat_template_tool_calls::known_marker_candidates::known_marker_candidates; let mistral_input = r#"[TOOL_CALLS]get_weather[ARGS]{"location":"Paris"}"#; @@ -369,7 +368,7 @@ mod tests { fn duck_type_resolves_gemma_input_via_paired_quote_shape() { use llama_cpp_bindings_types::ToolCallArguments; - use crate::chat_template_tool_calls::known_marker_candidates; + use crate::chat_template_tool_calls::known_marker_candidates::known_marker_candidates; let gemma_input = "<|tool_call>call:get_weather{location:<|\"|>Paris<|\"|>}"; diff --git a/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs b/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs index 3fb0e4367..66e5e9132 100644 --- a/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs +++ b/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs @@ -4,7 +4,7 @@ use llama_cpp_bindings_types::ParsedToolCallId; use llama_cpp_bindings_types::ToolCallArguments; use llama_cpp_bindings_types::ToolCallMarkers; -use crate::error::BracketedArgsFailure; +use crate::error::bracketed_args_failure::BracketedArgsFailure; use crate::tool_call_format::consume_optional_prefix::consume_optional_prefix; use crate::tool_call_format::parse_step::ParseStep; use crate::tool_call_format::separator_split::SeparatorSplit; @@ -106,7 +106,7 @@ mod tests { use serde_json::json; use super::parse; - use crate::error::BracketedArgsFailure; + use crate::error::bracketed_args_failure::BracketedArgsFailure; fn mistral3_markers() -> ToolCallMarkers { ToolCallMarkers { diff --git a/llama-cpp-bindings/src/tool_call_format/json_object.rs b/llama-cpp-bindings/src/tool_call_format/json_object.rs index 40be6d2ce..3755a249f 100644 --- a/llama-cpp-bindings/src/tool_call_format/json_object.rs +++ b/llama-cpp-bindings/src/tool_call_format/json_object.rs @@ -3,7 +3,7 @@ use llama_cpp_bindings_types::ParsedToolCall; use llama_cpp_bindings_types::ParsedToolCallId; use llama_cpp_bindings_types::ToolCallArguments; -use crate::error::JsonObjectFailure; +use crate::error::json_object_failure::JsonObjectFailure; fn try_parse_one_object( input: &str, @@ -85,7 +85,7 @@ mod tests { use serde_json::json; use super::parse; - use crate::error::JsonObjectFailure; + use crate::error::json_object_failure::JsonObjectFailure; fn qwen3_shape() -> JsonObjectShape { JsonObjectShape { diff --git a/llama-cpp-bindings/src/tool_call_format/key_value_xml_tags.rs b/llama-cpp-bindings/src/tool_call_format/key_value_xml_tags.rs index 55dcb3233..e1fc927dc 100644 --- a/llama-cpp-bindings/src/tool_call_format/key_value_xml_tags.rs +++ b/llama-cpp-bindings/src/tool_call_format/key_value_xml_tags.rs @@ -7,7 +7,7 @@ use nom::IResult; use nom::Parser; use nom::bytes::complete::take_until; -use crate::error::KeyValueXmlTagsFailure; +use crate::error::key_value_xml_tags_failure::KeyValueXmlTagsFailure; use crate::tool_call_format::parse_step::ParseStep; use crate::tool_call_format::scalar_value_to_json::scalar_value_to_json; @@ -176,7 +176,7 @@ mod tests { use serde_json::json; use super::parse; - use crate::error::KeyValueXmlTagsFailure; + use crate::error::key_value_xml_tags_failure::KeyValueXmlTagsFailure; fn glm47_markers() -> ToolCallMarkers { ToolCallMarkers { diff --git a/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs b/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs index 66801eb7b..1c89f6072 100644 --- a/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs +++ b/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs @@ -5,7 +5,7 @@ use llama_cpp_bindings_types::ToolCallArguments; use llama_cpp_bindings_types::ToolCallMarkers; use llama_cpp_bindings_types::ToolCallValueQuote; -use crate::error::PairedQuoteFailure; +use crate::error::paired_quote_failure::PairedQuoteFailure; use crate::tool_call_format::consume_optional_prefix::consume_optional_prefix; use crate::tool_call_format::parse_step::ParseStep; use crate::tool_call_format::scalar_value_to_json::scalar_value_to_json; @@ -216,7 +216,7 @@ mod tests { use serde_json::json; use super::parse; - use crate::error::PairedQuoteFailure; + use crate::error::paired_quote_failure::PairedQuoteFailure; fn gemma4_markers() -> ToolCallMarkers { ToolCallMarkers { diff --git a/llama-cpp-bindings/src/tool_call_format/tool_call_format_outcome.rs b/llama-cpp-bindings/src/tool_call_format/tool_call_format_outcome.rs index 1791071cf..e026529d6 100644 --- a/llama-cpp-bindings/src/tool_call_format/tool_call_format_outcome.rs +++ b/llama-cpp-bindings/src/tool_call_format/tool_call_format_outcome.rs @@ -1,6 +1,6 @@ use llama_cpp_bindings_types::ParsedToolCall; -use crate::error::ToolCallFormatFailure; +use crate::error::tool_call_format_failure::ToolCallFormatFailure; #[derive(Debug, Eq, PartialEq)] pub enum ToolCallFormatOutcome { diff --git a/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs b/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs index 7fe6eb96a..c4b822a4c 100644 --- a/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs +++ b/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs @@ -6,7 +6,7 @@ use nom::IResult; use nom::Parser; use nom::bytes::complete::take_until; -use crate::error::XmlFunctionTagsFailure; +use crate::error::xml_function_tags_failure::XmlFunctionTagsFailure; use crate::tool_call_format::scalar_value_to_json::scalar_value_to_json; const fn shape_is_complete(shape: &XmlTagsShape) -> bool { @@ -180,7 +180,7 @@ mod tests { use serde_json::json; use super::parse; - use crate::error::XmlFunctionTagsFailure; + use crate::error::xml_function_tags_failure::XmlFunctionTagsFailure; fn xml_shape() -> XmlTagsShape { XmlTagsShape { diff --git a/llama-cpp-test-harness/src/harness_run_error.rs b/llama-cpp-test-harness/src/harness_run_error.rs index fabee8e78..bd251072c 100644 --- a/llama-cpp-test-harness/src/harness_run_error.rs +++ b/llama-cpp-test-harness/src/harness_run_error.rs @@ -1,4 +1,4 @@ -use llama_cpp_bindings::error::LlamaCppError; +use llama_cpp_bindings::error::llama_cpp_error::LlamaCppError; use thiserror::Error; use crate::harness_arguments_error::HarnessArgumentsError; diff --git a/llama-cpp-test-harness/src/llama_fixture.rs b/llama-cpp-test-harness/src/llama_fixture.rs index f427f8aa4..150899b8f 100644 --- a/llama-cpp-test-harness/src/llama_fixture.rs +++ b/llama-cpp-test-harness/src/llama_fixture.rs @@ -4,7 +4,7 @@ use anyhow::Result; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::llama_backend::LlamaBackend; use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::mtmd::MtmdContext; +use llama_cpp_bindings::mtmd::mtmd_context::MtmdContext; use crate::context_params::ContextParams; diff --git a/llama-cpp-test-harness/src/load_key.rs b/llama-cpp-test-harness/src/load_key.rs index 2f889f49e..2e4473fef 100644 --- a/llama-cpp-test-harness/src/load_key.rs +++ b/llama-cpp-test-harness/src/load_key.rs @@ -30,7 +30,7 @@ impl LoadKey { mod tests { use std::path::PathBuf; - use llama_cpp_bindings::model::LlamaLoadMode; + use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; use crate::gguf_source::GgufSource; use crate::model_load_params::ModelLoadParams; diff --git a/llama-cpp-test-harness/src/phase_model_files.rs b/llama-cpp-test-harness/src/phase_model_files.rs index 6a467d47a..c898fbe64 100644 --- a/llama-cpp-test-harness/src/phase_model_files.rs +++ b/llama-cpp-test-harness/src/phase_model_files.rs @@ -4,8 +4,8 @@ use std::sync::Arc; use anyhow::Result; use llama_cpp_bindings::llama_backend::LlamaBackend; use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::mtmd::MtmdContext; -use llama_cpp_bindings::mtmd::MtmdContextParams; +use llama_cpp_bindings::mtmd::mtmd_context::MtmdContext; +use llama_cpp_bindings::mtmd::mtmd_context_params::MtmdContextParams; use crate::model_load_params::ModelLoadParams; use crate::phase_state::PhaseState; @@ -61,9 +61,9 @@ mod tests { use std::sync::Arc; use std::sync::PoisonError; - use llama_cpp_bindings::LlamaModelLoadError; + use llama_cpp_bindings::error::llama_model_load_error::LlamaModelLoadError; use llama_cpp_bindings::llama_backend::LlamaBackend; - use llama_cpp_bindings::model::LlamaLoadMode; + use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; use crate::model_load_params::ModelLoadParams; use crate::test_backend_gate::BACKEND_INIT_GATE; diff --git a/llama-cpp-test-harness/src/phase_setup.rs b/llama-cpp-test-harness/src/phase_setup.rs index 9b70eade2..59d485eae 100644 --- a/llama-cpp-test-harness/src/phase_setup.rs +++ b/llama-cpp-test-harness/src/phase_setup.rs @@ -57,7 +57,7 @@ mod tests { use std::sync::PoisonError; use llama_cpp_bindings::llama_backend::LlamaBackend; - use llama_cpp_bindings::model::LlamaLoadMode; + use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; use crate::gguf_source::GgufSource; use crate::load_key::LoadKey; diff --git a/llama-cpp-test-harness/src/phase_state.rs b/llama-cpp-test-harness/src/phase_state.rs index 53e4560f3..1e70a10d9 100644 --- a/llama-cpp-test-harness/src/phase_state.rs +++ b/llama-cpp-test-harness/src/phase_state.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use llama_cpp_bindings::llama_backend::LlamaBackend; use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::mtmd::MtmdContext; +use llama_cpp_bindings::mtmd::mtmd_context::MtmdContext; #[derive(Debug)] pub struct PhaseState { diff --git a/llama-cpp-test-harness/src/run.rs b/llama-cpp-test-harness/src/run.rs index afb674dc7..89b20f213 100644 --- a/llama-cpp-test-harness/src/run.rs +++ b/llama-cpp-test-harness/src/run.rs @@ -30,7 +30,7 @@ mod tests { use libtest_mimic::Arguments; use libtest_mimic::Conclusion; - use llama_cpp_bindings::error::LlamaCppError; + use llama_cpp_bindings::error::llama_cpp_error::LlamaCppError; use llama_cpp_bindings::llama_backend::LlamaBackend; use crate::harness_run_error::HarnessRunError; diff --git a/llama-cpp-test-harness/src/scheduled_phase.rs b/llama-cpp-test-harness/src/scheduled_phase.rs index c743fcce4..b233482cb 100644 --- a/llama-cpp-test-harness/src/scheduled_phase.rs +++ b/llama-cpp-test-harness/src/scheduled_phase.rs @@ -92,7 +92,7 @@ mod tests { use libtest_mimic::Arguments; use llama_cpp_bindings::llama_backend::LlamaBackend; - use llama_cpp_bindings::model::LlamaLoadMode; + use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; use crate::context_params::ContextParams; use crate::execution_phase::ExecutionPhase; From ae0fac0370f6a9375c4eadd1ee178aa929a4a573 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sat, 31 Oct 2026 05:06:10 +0100 Subject: [PATCH 24/82] Add batch entries in one wrapper call, cache context limits and the vocab, decode pieces without dropping bytes, and read chat spans as one array --- .../src/binding_callbacks.rs | 23 +- llama-cpp-bindings-sys/wrapper_batch.cpp | 132 +++++++---- llama-cpp-bindings-sys/wrapper_batch.h | 55 +++-- .../wrapper_chat_session.cpp | 75 +++---- llama-cpp-bindings-sys/wrapper_chat_session.h | 22 +- .../wrapper_fault_injection.h | 2 +- .../tests/decode_batch_rejection.rs | 5 +- .../tests/detokenization.rs | 36 +++ .../tests/encode_batch_rejection.rs | 5 +- llama-cpp-bindings/src/chat_session.rs | 26 +-- llama-cpp-bindings/src/chat_span_kind.rs | 4 +- llama-cpp-bindings/src/chat_spans.rs | 29 +-- llama-cpp-bindings/src/context.rs | 207 ++++++++++-------- .../src/context/context_limits.rs | 9 +- .../src/context/decoded_outputs.rs | 2 +- llama-cpp-bindings/src/lib.rs | 1 - llama-cpp-bindings/src/llama_batch.rs | 50 +++-- llama-cpp-bindings/src/llama_batch_entry.rs | 11 - llama-cpp-bindings/src/model.rs | 82 +++++-- .../src/mtmd/mtmd_eval_limits.rs | 2 +- 20 files changed, 461 insertions(+), 317 deletions(-) delete mode 100644 llama-cpp-bindings/src/llama_batch_entry.rs diff --git a/llama-cpp-bindings-build/src/binding_callbacks.rs b/llama-cpp-bindings-build/src/binding_callbacks.rs index 91ed27767..c4cec90f5 100644 --- a/llama-cpp-bindings-build/src/binding_callbacks.rs +++ b/llama-cpp-bindings-build/src/binding_callbacks.rs @@ -14,12 +14,16 @@ use crate::build_error::BuildError; use crate::cargo_directive::CargoDirective; use crate::privatized_field::PrivatizedField; -const COMPARED_STRUCTS: &[&str] = &["llama_logit_bias", "llama_token_data"]; +const COMPARED_STRUCTS_WITH_FLOATS: &[&str] = &["llama_logit_bias", "llama_token_data"]; + +const COMPARED_STRUCTS_WITHOUT_FLOATS: &[&str] = &["llama_rs_batch_entry"]; const SUPERSEDED_ENUM_VARIANTS: &[&str] = &["GGML_PREC_DEFAULT"]; fn derives_for(type_name: &str) -> Vec { - if COMPARED_STRUCTS.contains(&type_name) { + if COMPARED_STRUCTS_WITHOUT_FLOATS.contains(&type_name) { + vec!["PartialEq".to_owned(), "Eq".to_owned()] + } else if COMPARED_STRUCTS_WITH_FLOATS.contains(&type_name) { vec!["PartialEq".to_owned()] } else { Vec::new() @@ -33,8 +37,8 @@ fn variant_behavior_for(variant_name: &str) -> Option } /// Observes bindgen while it parses: keeps the [`PrivatizedField::ALL`] fields private, hides -/// enum variants llama.cpp superseded, derives `PartialEq` for the structs the bindings compare, -/// and records what the bindings depend on. +/// enum variants llama.cpp superseded, derives the equality the fields of each struct the bindings +/// compare allow, and records what the bindings depend on. #[derive(Clone, Debug)] pub struct BindingCallbacks { privatized_field_hits: Rc<[Cell]>, @@ -141,12 +145,19 @@ mod tests { } #[test] - fn only_the_structs_the_bindings_compare_derive_partial_eq() { + fn the_structs_the_bindings_compare_derive_equality_their_fields_allow() { assert_eq!( - ["llama_token_data", "llama_logit_bias", "llama_sampler_i"].map(derives_for), + [ + "llama_token_data", + "llama_logit_bias", + "llama_rs_batch_entry", + "llama_sampler_i" + ] + .map(derives_for), [ vec!["PartialEq".to_owned()], vec!["PartialEq".to_owned()], + vec!["PartialEq".to_owned(), "Eq".to_owned()], Vec::new() ] ); diff --git a/llama-cpp-bindings-sys/wrapper_batch.cpp b/llama-cpp-bindings-sys/wrapper_batch.cpp index 6c2e51d61..08148a471 100644 --- a/llama-cpp-bindings-sys/wrapper_batch.cpp +++ b/llama-cpp-bindings-sys/wrapper_batch.cpp @@ -43,15 +43,65 @@ extern "C" auto llama_rs_batch_ext_init( } } -extern "C" auto llama_rs_batch_ext_add_entry( +namespace { + +auto add_entry( struct llama_batch_ext * batch, - llama_token token, - llama_pos pos, + const llama_rs_batch_entry & entry, + const gsl::span seq_ids, + int32_t & out_llama_cpp_return_code) -> llama_rs_batch_ext_assign_entries_status { + if (seq_ids.empty()) { + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_EMPTY_SEQ_IDS; + } + + const int32_t index = llama_batch_ext_add_token(batch, seq_ids[0], entry.token); + + if (index < 0) { + out_llama_cpp_return_code = index; + + switch (index) { + case -1: + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_BATCH_FULL; + case -2: + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_TOKEN; + case -3: + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_SEQUENCE_ID; + default: + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_RETURNED_UNKNOWN_CODE; + } + } + + const gsl::span additional_seq_ids = seq_ids.subspan(1); + const bool every_seq_id_added = std::all_of( + additional_seq_ids.begin(), additional_seq_ids.end(), [&](const llama_seq_id seq_id) -> bool { + return llama_batch_ext_add_seq(batch, index, seq_id); + }); + + if (!every_seq_id_added) { + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_SEQUENCE_ID; + } + + if (!llama_batch_ext_set_pos(batch, index, &entry.pos)) { + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_POSITION_REJECTED; + } + + if (!llama_batch_ext_set_output_logits(batch, index, entry.output)) { + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OUTPUT_REJECTED; + } + + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OK; +} + +} // namespace + +extern "C" auto llama_rs_batch_ext_assign_entries( + struct llama_batch_ext * batch, + const llama_rs_batch_entry * entries, + size_t entries_count, const llama_seq_id * seq_ids, - size_t n_seq_ids, - bool output, + size_t seq_ids_count, int32_t * out_llama_cpp_return_code, - char ** out_error) -> llama_rs_batch_ext_add_entry_status { + char ** out_error) -> llama_rs_batch_ext_assign_entries_status { if (out_llama_cpp_return_code != nullptr) { *out_llama_cpp_return_code = 0; } @@ -59,65 +109,51 @@ extern "C" auto llama_rs_batch_ext_add_entry( *out_error = nullptr; } if (batch == nullptr) { - return LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_BATCH_ARG; + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_BATCH_ARG; + } + if (entries == nullptr) { + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_ENTRIES_ARG; } if (seq_ids == nullptr) { - return LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_SEQ_IDS_ARG; + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_SEQ_IDS_ARG; } if (out_llama_cpp_return_code == nullptr) { - return LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG; + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG; } if (out_error == nullptr) { - return LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_OUT_ERROR_ARG; - } - if (n_seq_ids == 0) { - return LLAMA_RS_BATCH_EXT_ADD_ENTRY_EMPTY_SEQ_IDS; + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_ERROR_ARG; } try { - llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_BATCH_EXT_ADD_ENTRY_THROWS); - const gsl::span seq_id_span(seq_ids, n_seq_ids); - const int32_t index = llama_batch_ext_add_token(batch, seq_id_span[0], token); - - if (index < 0) { - *out_llama_cpp_return_code = index; - - switch (index) { - case -1: - return LLAMA_RS_BATCH_EXT_ADD_ENTRY_BATCH_FULL; - case -2: - return LLAMA_RS_BATCH_EXT_ADD_ENTRY_INVALID_TOKEN; - case -3: - return LLAMA_RS_BATCH_EXT_ADD_ENTRY_INVALID_SEQUENCE_ID; - default: - return LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_RETURNED_UNKNOWN_CODE; - } - } + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_BATCH_EXT_ASSIGN_ENTRIES_THROWS); + llama_batch_ext_clear(batch); - const gsl::span additional_seq_ids = seq_id_span.subspan(1); - const bool every_seq_id_added = std::all_of( - additional_seq_ids.begin(), additional_seq_ids.end(), [&](const llama_seq_id seq_id) -> bool { - return llama_batch_ext_add_seq(batch, index, seq_id); - }); + const gsl::span entry_span(entries, entries_count); + const gsl::span seq_id_span(seq_ids, seq_ids_count); - if (!every_seq_id_added) { - return LLAMA_RS_BATCH_EXT_ADD_ENTRY_INVALID_SEQUENCE_ID; - } + for (const llama_rs_batch_entry & entry : entry_span) { + if (entry.seq_ids_offset > seq_id_span.size() || + entry.seq_ids_count > seq_id_span.size() - entry.seq_ids_offset) { + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_SEQ_IDS_OUT_OF_RANGE; + } - if (!llama_batch_ext_set_pos(batch, index, &pos)) { - return LLAMA_RS_BATCH_EXT_ADD_ENTRY_POSITION_REJECTED; - } + const llama_rs_batch_ext_assign_entries_status status = add_entry( + batch, + entry, + seq_id_span.subspan(entry.seq_ids_offset, entry.seq_ids_count), + *out_llama_cpp_return_code); - if (!llama_batch_ext_set_output_logits(batch, index, output)) { - return LLAMA_RS_BATCH_EXT_ADD_ENTRY_OUTPUT_REJECTED; + if (status != LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OK) { + return status; + } } - return LLAMA_RS_BATCH_EXT_ADD_ENTRY_OK; + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OK; } catch (...) { return llama_rs_report_current_exception( out_error, - LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_BATCH_EXT_ADD_ENTRY_ERROR_STRING_ALLOCATION_FAILED); + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_ERROR_STRING_ALLOCATION_FAILED); } } diff --git a/llama-cpp-bindings-sys/wrapper_batch.h b/llama-cpp-bindings-sys/wrapper_batch.h index 396aa9468..5c3d9acac 100644 --- a/llama-cpp-bindings-sys/wrapper_batch.h +++ b/llama-cpp-bindings-sys/wrapper_batch.h @@ -25,31 +25,40 @@ llama_rs_batch_ext_init_status llama_rs_batch_ext_init( struct llama_batch_ext ** out_batch, char ** out_error); -typedef enum llama_rs_batch_ext_add_entry_status { - LLAMA_RS_BATCH_EXT_ADD_ENTRY_OK = 0, - LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_BATCH_ARG, - LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_SEQ_IDS_ARG, - LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG, - LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_OUT_ERROR_ARG, - LLAMA_RS_BATCH_EXT_ADD_ENTRY_EMPTY_SEQ_IDS, - LLAMA_RS_BATCH_EXT_ADD_ENTRY_BATCH_FULL, - LLAMA_RS_BATCH_EXT_ADD_ENTRY_INVALID_TOKEN, - LLAMA_RS_BATCH_EXT_ADD_ENTRY_INVALID_SEQUENCE_ID, - LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_RETURNED_UNKNOWN_CODE, - LLAMA_RS_BATCH_EXT_ADD_ENTRY_POSITION_REJECTED, - LLAMA_RS_BATCH_EXT_ADD_ENTRY_OUTPUT_REJECTED, - LLAMA_RS_BATCH_EXT_ADD_ENTRY_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_batch_ext_add_entry_status; - -llama_rs_batch_ext_add_entry_status llama_rs_batch_ext_add_entry( +typedef struct llama_rs_batch_entry { + llama_token token; + llama_pos pos; + size_t seq_ids_offset; + size_t seq_ids_count; + bool output; +} llama_rs_batch_entry; + +typedef enum llama_rs_batch_ext_assign_entries_status { + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OK = 0, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_BATCH_ARG, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_ENTRIES_ARG, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_SEQ_IDS_ARG, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_ERROR_ARG, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_SEQ_IDS_OUT_OF_RANGE, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_EMPTY_SEQ_IDS, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_BATCH_FULL, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_TOKEN, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_SEQUENCE_ID, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_RETURNED_UNKNOWN_CODE, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_POSITION_REJECTED, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OUTPUT_REJECTED, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_batch_ext_assign_entries_status; + +llama_rs_batch_ext_assign_entries_status llama_rs_batch_ext_assign_entries( struct llama_batch_ext * batch, - llama_token token, - llama_pos pos, + const llama_rs_batch_entry * entries, + size_t entries_count, const llama_seq_id * seq_ids, - size_t n_seq_ids, - bool output, + size_t seq_ids_count, int32_t * out_llama_cpp_return_code, char ** out_error); diff --git a/llama-cpp-bindings-sys/wrapper_chat_session.cpp b/llama-cpp-bindings-sys/wrapper_chat_session.cpp index 10e5df6d9..b0174880b 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_session.cpp +++ b/llama-cpp-bindings-sys/wrapper_chat_session.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -33,12 +34,7 @@ struct llama_rs_chat_session { std::vector generation_prompt_tokens; common_params_sampling sampling; std::vector preserved_tokens; -}; - -struct llama_rs_chat_span { - llama_rs_chat_span_kind kind; - size_t start; - size_t end; + std::unordered_set trigger_rules; }; struct llama_rs_chat_spans { @@ -100,33 +96,44 @@ auto span_kind_of_tag(const std::string & tag) -> std::optional bool { - if (rule.empty() || !parser.has_rule(rule)) { - return false; - } +auto trigger_rules_of(const common_peg_arena & parser) -> std::unordered_set { + std::unordered_set trigger_rules; - const auto * rule_parser = std::get_if(&parser.get(parser.get_rule(rule))); + for (common_peg_parser_id parser_id = 0; parser_id < parser.size(); ++parser_id) { + const auto * rule_parser = std::get_if(&parser.get(parser_id)); + if (rule_parser != nullptr && rule_parser->trigger && !rule_parser->name.empty() && + parser.get_rule(rule_parser->name) == parser_id) { + trigger_rules.insert(rule_parser->name); + } + } - return rule_parser != nullptr && rule_parser->trigger; + return trigger_rules; } -auto span_kind_of_node(const common_peg_arena & parser, const common_peg_ast_node & node) +auto span_kind_of_node(const std::unordered_set & trigger_rules, const common_peg_ast_node & node) -> std::optional { if (const auto kind = span_kind_of_tag(node.tag)) { return kind; } - if (is_trigger_rule(parser, node.rule)) { + if (trigger_rules.find(node.rule) != trigger_rules.end()) { return LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL_MARKER; } return std::nullopt; } -auto chat_input_of(const char * text, size_t text_length, const llama_token * byte_tokens) -> common_chat_input { - common_chat_input input; - input.text.assign(text, text_length); +auto chat_input_of( + const common_chat_input & prefix, + const char * text, + size_t text_length, + const llama_token * byte_tokens) -> common_chat_input { const gsl::span tokens(byte_tokens, text_length); - input.tokens.assign(tokens.begin(), tokens.end()); + common_chat_input input; + input.text.reserve(prefix.text.size() + text_length); + input.text.append(prefix.text).append(text, text_length); + input.tokens.reserve(prefix.tokens.size() + text_length); + input.tokens.insert(input.tokens.end(), prefix.tokens.begin(), prefix.tokens.end()); + input.tokens.insert(input.tokens.end(), tokens.begin(), tokens.end()); return input; } @@ -229,6 +236,7 @@ extern "C" auto llama_rs_chat_session_create( chat_session->session.apply_sampling(chat_session->sampling); chat_session->preserved_tokens.assign( chat_session->sampling.preserved_tokens.begin(), chat_session->sampling.preserved_tokens.end()); + chat_session->trigger_rules = trigger_rules_of(session_parser(chat_session->session)); *out_session = chat_session.release(); @@ -394,7 +402,10 @@ extern "C" auto llama_rs_chat_session_parse( try { parsed->message = common_chat_peg_parse( - session->session.parser(), chat_input_of(text, text_length, byte_tokens), is_partial, parser_params); + session->session.parser(), + chat_input_of(common_chat_input(), text, text_length, byte_tokens), + is_partial, + parser_params); } catch (const std::bad_alloc &) { throw; } catch (const std::runtime_error & err) { @@ -448,9 +459,7 @@ extern "C" auto llama_rs_chat_session_classify( try { llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_CHAT_SESSION_CLASSIFY_THROWS); - common_chat_input parse_input = chat_input_of(text, text_length, byte_tokens); - parse_input.prepend(session->generation_prompt); - + common_chat_input parse_input = chat_input_of(session->generation_prompt, text, text_length, byte_tokens); const size_t generation_prompt_length = session->generation_prompt.size(); common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_LENIENT; @@ -470,7 +479,7 @@ extern "C" auto llama_rs_chat_session_classify( spans->parsed_length = result.end > generation_prompt_length ? result.end - generation_prompt_length : 0; context.ast.visit(result, [&](const common_peg_ast_node & node) -> void { - const auto kind = span_kind_of_node(parser, node); + const auto kind = span_kind_of_node(session->trigger_rules, node); if (!kind || node.end <= generation_prompt_length) { return; } @@ -493,24 +502,8 @@ extern "C" auto llama_rs_chat_session_classify( } } -extern "C" auto llama_rs_chat_spans_parsed_length(llama_rs_chat_spans_handle spans) -> size_t { - return spans->parsed_length; -} - -extern "C" auto llama_rs_chat_spans_count(llama_rs_chat_spans_handle spans) -> size_t { - return spans->spans.size(); -} - -extern "C" auto llama_rs_chat_spans_kind(llama_rs_chat_spans_handle spans, size_t index) -> llama_rs_chat_span_kind { - return spans->spans.at(index).kind; -} - -extern "C" auto llama_rs_chat_spans_start(llama_rs_chat_spans_handle spans, size_t index) -> size_t { - return spans->spans.at(index).start; -} - -extern "C" auto llama_rs_chat_spans_end(llama_rs_chat_spans_handle spans, size_t index) -> size_t { - return spans->spans.at(index).end; +extern "C" auto llama_rs_chat_spans_view_of(llama_rs_chat_spans_handle spans) -> llama_rs_chat_spans_view { + return {spans->spans.data(), spans->spans.size(), spans->parsed_length}; } extern "C" auto llama_rs_chat_spans_free(llama_rs_chat_spans_handle spans) -> void { diff --git a/llama-cpp-bindings-sys/wrapper_chat_session.h b/llama-cpp-bindings-sys/wrapper_chat_session.h index 3169dc1a1..4cbeedc3d 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_session.h +++ b/llama-cpp-bindings-sys/wrapper_chat_session.h @@ -150,6 +150,18 @@ typedef enum llama_rs_chat_span_kind { LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL_MARKER, } llama_rs_chat_span_kind; +typedef struct llama_rs_chat_span { + llama_rs_chat_span_kind kind; + size_t start; + size_t end; +} llama_rs_chat_span; + +typedef struct llama_rs_chat_spans_view { + const llama_rs_chat_span * spans; + size_t spans_count; + size_t parsed_length; +} llama_rs_chat_spans_view; + typedef enum llama_rs_chat_session_classify_status { LLAMA_RS_CHAT_SESSION_CLASSIFY_OK = 0, LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_SESSION_ARG, @@ -172,15 +184,7 @@ llama_rs_chat_session_classify_status llama_rs_chat_session_classify( llama_rs_chat_spans_handle * out_spans, char ** out_error); -size_t llama_rs_chat_spans_parsed_length(llama_rs_chat_spans_handle spans); - -size_t llama_rs_chat_spans_count(llama_rs_chat_spans_handle spans); - -llama_rs_chat_span_kind llama_rs_chat_spans_kind(llama_rs_chat_spans_handle spans, size_t index); - -size_t llama_rs_chat_spans_start(llama_rs_chat_spans_handle spans, size_t index); - -size_t llama_rs_chat_spans_end(llama_rs_chat_spans_handle spans, size_t index); +llama_rs_chat_spans_view llama_rs_chat_spans_view_of(llama_rs_chat_spans_handle spans); void llama_rs_chat_spans_free(llama_rs_chat_spans_handle spans); diff --git a/llama-cpp-bindings-sys/wrapper_fault_injection.h b/llama-cpp-bindings-sys/wrapper_fault_injection.h index 3adc2009a..7a13381ea 100644 --- a/llama-cpp-bindings-sys/wrapper_fault_injection.h +++ b/llama-cpp-bindings-sys/wrapper_fault_injection.h @@ -9,8 +9,8 @@ extern "C" { typedef enum llama_rs_fault_point { LLAMA_RS_FAULT_POINT_NONE = 0, + LLAMA_RS_FAULT_POINT_BATCH_EXT_ASSIGN_ENTRIES_THROWS, LLAMA_RS_FAULT_POINT_BATCH_EXT_INIT_THROWS, - LLAMA_RS_FAULT_POINT_BATCH_EXT_ADD_ENTRY_THROWS, LLAMA_RS_FAULT_POINT_CHAT_SESSION_CLASSIFY_THROWS, LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_NOT_UTF8, LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_TRIGGER_TYPE_UNRECOGNIZED, diff --git a/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs b/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs index 0d309fcb3..19456bc3b 100644 --- a/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs +++ b/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs @@ -123,8 +123,9 @@ fn decoding_reports_a_token_llama_cpp_cannot_add_to_its_batch( .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - let _fault = - ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_BATCH_EXT_ADD_ENTRY_THROWS); + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_BATCH_EXT_ASSIGN_ENTRIES_THROWS, + ); assert_eq!( context.decode(&batch), diff --git a/llama-cpp-bindings-tests/tests/detokenization.rs b/llama-cpp-bindings-tests/tests/detokenization.rs index d54bae55e..78ad55f57 100644 --- a/llama-cpp-bindings-tests/tests/detokenization.rs +++ b/llama-cpp-bindings-tests/tests/detokenization.rs @@ -1,5 +1,6 @@ use std::num::NonZeroU16; +use anyhow::Context as _; use anyhow::Result; use llama_cpp_bindings::error::token_to_string_error::TokenToStringError; use llama_cpp_bindings::model::add_bos::AddBos; @@ -348,3 +349,38 @@ fn token_to_piece_bytes_refuses_a_buffer_beyond_a_c_int_before_allocating( .map(TokenToStringError::IntConversionError) ); } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_to_piece_replaces_a_lone_continuation_byte_with_the_replacement_character( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let continuation_byte_token = (0..model.n_vocab()) + .map(LlamaToken::new) + .find(|token| { + model + .token_to_piece_bytes(*token, 8, true, None) + .is_ok_and(|bytes| bytes == [0x80]) + }) + .context("the byte-level vocabulary has a token for the byte 0x80")?; + let mut decoder = encoding_rs::UTF_8.new_decoder(); + + assert_eq!( + model.token_to_piece( + &SampledToken::Content(continuation_byte_token), + &mut decoder, + true, + None + )?, + "\u{FFFD}" + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs b/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs index d1ac0dbe3..24509fc55 100644 --- a/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs +++ b/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs @@ -61,8 +61,9 @@ fn encoding_reports_a_token_llama_cpp_cannot_add_to_its_batch( .str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - let _fault = - ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_BATCH_EXT_ADD_ENTRY_THROWS); + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_BATCH_EXT_ASSIGN_ENTRIES_THROWS, + ); assert_eq!( context.encode(&batch), diff --git a/llama-cpp-bindings/src/chat_session.rs b/llama-cpp-bindings/src/chat_session.rs index 8083bf544..a13154cb6 100644 --- a/llama-cpp-bindings/src/chat_session.rs +++ b/llama-cpp-bindings/src/chat_session.rs @@ -2,6 +2,7 @@ use std::ffi::CString; use std::ffi::c_char; use std::ptr; use std::ptr::NonNull; +use std::slice; use llama_cpp_bindings_types::ParsedChatMessage; use llama_cpp_ffi_status::read_and_free_cpp_string; @@ -243,24 +244,21 @@ fn read_chat_spans( spans: NonNull, completeness: ChatOutputCompleteness, ) -> Result { - let count = unsafe { llama_cpp_bindings_sys::llama_rs_chat_spans_count(spans.as_ptr()) }; - let read = (0..count) - .map(|index| { - let bytes = unsafe { - llama_cpp_bindings_sys::llama_rs_chat_spans_start(spans.as_ptr(), index) - ..llama_cpp_bindings_sys::llama_rs_chat_spans_end(spans.as_ptr(), index) - }; - - ChatSpanKind::try_from(unsafe { - llama_cpp_bindings_sys::llama_rs_chat_spans_kind(spans.as_ptr(), index) + let view = unsafe { llama_cpp_bindings_sys::llama_rs_chat_spans_view_of(spans.as_ptr()) }; + let wrapper_spans = NonNull::new(view.spans.cast_mut()).map_or(&[][..], |first_span| unsafe { + slice::from_raw_parts(first_span.as_ptr(), view.spans_count) + }); + let read = wrapper_spans + .iter() + .map(|wrapper_span| { + ChatSpanKind::try_from(wrapper_span.kind).map(|kind| ChatSpan { + kind, + bytes: wrapper_span.start..wrapper_span.end, }) - .map(|kind| ChatSpan { kind, bytes }) }) .collect::, llama_cpp_ffi_status::FfiStatusError>>() .map(|spans_read| ChatSpans { - parsed_length: unsafe { - llama_cpp_bindings_sys::llama_rs_chat_spans_parsed_length(spans.as_ptr()) - }, + parsed_length: view.parsed_length, spans: spans_read, completeness, }) diff --git a/llama-cpp-bindings/src/chat_span_kind.rs b/llama-cpp-bindings/src/chat_span_kind.rs index 86f8b264f..96403275a 100644 --- a/llama-cpp-bindings/src/chat_span_kind.rs +++ b/llama-cpp-bindings/src/chat_span_kind.rs @@ -40,7 +40,7 @@ impl TryFrom for ChatSpanKind { Ok(Self::ToolCallMarker) } other => Err(llama_cpp_ffi_status::FfiStatusError { - operation: "llama_rs_chat_spans_kind", + operation: "llama_rs_chat_spans_view_of", code: i64::from(other), }), } @@ -108,7 +108,7 @@ mod tests { assert_eq!( ChatSpanKind::try_from(99), Err(llama_cpp_ffi_status::FfiStatusError { - operation: "llama_rs_chat_spans_kind", + operation: "llama_rs_chat_spans_view_of", code: 99, }) ); diff --git a/llama-cpp-bindings/src/chat_spans.rs b/llama-cpp-bindings/src/chat_spans.rs index 9e8f55d6f..cb8031b0c 100644 --- a/llama-cpp-bindings/src/chat_spans.rs +++ b/llama-cpp-bindings/src/chat_spans.rs @@ -55,25 +55,20 @@ impl ChatSpans { return None; } - let roles = bytes + bytes .clone() - .map(|byte| self.tagged_role_at(byte)) - .collect::>>(); - - if let Some(text_role) = roles - .iter() - .flatten() + .filter_map(|byte| self.tagged_role_at(byte)) .find(|role| matches!(role, ChatPieceRole::Text(_))) - { - return Some(*text_role); - } - - match roles.first() { - Some(Some(marker_role)) => Some(*marker_role), - Some(None) | None => self - .untagged_marker_section(bytes.start) - .map(ChatPieceRole::Marker), - } + .or_else(|| { + bytes + .clone() + .next() + .and_then(|first_byte| self.tagged_role_at(first_byte)) + }) + .or_else(|| { + self.untagged_marker_section(bytes.start) + .map(ChatPieceRole::Marker) + }) } } diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index cb787621b..27ae33f20 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -220,71 +220,79 @@ fn batch_ext_init_status_to_result( } } -fn batch_ext_add_entry_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_batch_ext_add_entry_status, +fn batch_ext_assign_entries_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_batch_ext_assign_entries_status, out_error: *mut c_char, ) -> Result<(), BatchTransferError> { let refused_validated_entry = |detail: &'static str| -> Result<(), BatchTransferError> { Err(llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_batch_ext_add_entry", + operation: "llama_rs_batch_ext_assign_entries", detail, } .into()) }; match status { - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_ERROR_STRING_ALLOCATION_FAILED => { + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OK => Ok(()), + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_ERROR_STRING_ALLOCATION_FAILED => { Err(BatchTransferError::NotEnoughMemory) } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_OUT_OF_MEMORY => { + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_OUT_OF_MEMORY => { Err(BatchTransferError::LlamaCppOutOfMemory) } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_THREW_CXX_EXCEPTION => { + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_THREW_CXX_EXCEPTION => { let message = unsafe { read_and_free_cpp_string( out_error, - "llama_rs_batch_ext_add_entry", + "llama_rs_batch_ext_assign_entries", "reported a thrown C++ exception without an error message", ) }?; Err(BatchTransferError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_BATCH_FULL => refused_validated_entry( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_BATCH_FULL => refused_validated_entry( "refused an entry of a batch validated against its limits as full", ), - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_INVALID_TOKEN => { + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_TOKEN => { refused_validated_entry("refused a token validated against the vocabulary") } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_INVALID_SEQUENCE_ID => { + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_SEQUENCE_ID => { refused_validated_entry("refused a sequence id validated against the context") } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_RETURNED_UNKNOWN_CODE => { + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_RETURNED_UNKNOWN_CODE => { refused_validated_entry("refused an entry with an undocumented code") } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_POSITION_REJECTED => { + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_POSITION_REJECTED => { refused_validated_entry("refused the position of an entry it had just added") } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_OUTPUT_REJECTED => { + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OUTPUT_REJECTED => { refused_validated_entry("refused the output flag of an entry it had just added") } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_EMPTY_SEQ_IDS => { + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_EMPTY_SEQ_IDS => { refused_validated_entry("was given an entry without sequence ids") } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_BATCH_ARG => { + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_SEQ_IDS_OUT_OF_RANGE => { + refused_validated_entry( + "was given an entry whose sequence ids lie outside the sequence id buffer", + ) + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_ENTRIES_ARG => { + refused_validated_entry("was given a null entries argument") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_BATCH_ARG => { refused_validated_entry("was given a null batch argument") } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_SEQ_IDS_ARG => { + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_SEQ_IDS_ARG => { refused_validated_entry("was given a null seq_ids argument") } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG => { + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG => { refused_validated_entry("was given a null out_llama_cpp_return_code argument") } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_OUT_ERROR_ARG => { + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_ERROR_ARG => { refused_validated_entry("was given a null out_error argument") } other => Err(llama_cpp_ffi_status::FfiStatusError { - operation: "llama_rs_batch_ext_add_entry", + operation: "llama_rs_batch_ext_assign_entries", code: i64::from(other), } .into()), @@ -395,6 +403,8 @@ pub struct LlamaContext<'model> { abort_flag: Option>, decoded_outputs: DecodedOutputs, batch_ext: NonNull, + limits: ContextLimits, + decodes_batches_in_one_micro_batch: bool, } impl Debug for LlamaContext<'_> { @@ -427,6 +437,10 @@ impl<'model> LlamaContext<'model> { let pooling_type = LlamaPoolingType::from(unsafe { llama_cpp_bindings_sys::llama_pooling_type(context.as_ptr()) }); + let keeps_no_memory = + unsafe { llama_cpp_bindings_sys::llama_get_memory(context.as_ptr()) }.is_null(); + let attends_causally = + unsafe { llama_cpp_bindings_sys::llama_get_causal_attn(context.as_ptr()) }; Ok(Self { context, @@ -437,6 +451,13 @@ impl<'model> LlamaContext<'model> { pooling_type, }, batch_ext, + limits: ContextLimits { + n_batch: unsafe { llama_cpp_bindings_sys::llama_n_batch(context.as_ptr()) }, + n_ubatch: unsafe { llama_cpp_bindings_sys::llama_n_ubatch(context.as_ptr()) }, + n_seq_max: unsafe { llama_cpp_bindings_sys::llama_n_seq_max(context.as_ptr()) }, + n_vocab: model.n_vocab(), + }, + decodes_batches_in_one_micro_batch: keeps_no_memory || !attends_causally, }) } @@ -484,26 +505,18 @@ impl<'model> LlamaContext<'model> { } #[must_use] - pub fn n_batch(&self) -> u32 { - unsafe { llama_cpp_bindings_sys::llama_n_batch(self.context.as_ptr()) } + pub const fn n_batch(&self) -> u32 { + self.limits.n_batch } #[must_use] - pub fn n_ubatch(&self) -> u32 { - unsafe { llama_cpp_bindings_sys::llama_n_ubatch(self.context.as_ptr()) } + pub const fn n_ubatch(&self) -> u32 { + self.limits.n_ubatch } #[must_use] - pub fn n_seq_max(&self) -> u32 { - unsafe { llama_cpp_bindings_sys::llama_n_seq_max(self.context.as_ptr()) } - } - - fn decodes_batches_in_one_micro_batch(&self) -> bool { - let keeps_no_memory = !self.has_memory(); - let attends_causally = - unsafe { llama_cpp_bindings_sys::llama_get_causal_attn(self.context.as_ptr()) }; - - keeps_no_memory || !attends_causally + pub const fn n_seq_max(&self) -> u32 { + self.limits.n_seq_max } fn exceeds_micro_batch(&self, batch: &LlamaBatch) -> bool { @@ -556,44 +569,42 @@ impl<'model> LlamaContext<'model> { /// Records which rows the last decode wrote, for a decode llama.cpp ran on this context /// without one of this context's batches, such as the mtmd helper decoding media chunks. pub fn record_decoded_rows(&mut self, rows: DecodedOutputRows) { - self.decoded_outputs = DecodedOutputs { - rows, - pooling_type: self.pooling_type(), - }; + self.decoded_outputs.rows = rows; } - fn transfer_batch(&self, batch: &LlamaBatch) -> Result<(), BatchTransferError> { - let context_limits = ContextLimits::of(self); + fn record_batch_output_rows(&mut self, output_token_indices: &[i32]) { + if let DecodedOutputRows::BatchIndices(rows) = &mut self.decoded_outputs.rows { + rows.clear(); + rows.extend_from_slice(output_token_indices); + } else { + self.decoded_outputs.rows = + DecodedOutputRows::BatchIndices(output_token_indices.to_vec()); + } + } - if batch.limits() != context_limits { + fn transfer_batch(&self, batch: &LlamaBatch) -> Result<(), BatchTransferError> { + if batch.limits() != self.limits { return Err(BatchTransferError::LimitsMismatch { batch: batch.limits(), - context: context_limits, + context: self.limits, }); } - unsafe { llama_cpp_bindings_sys::llama_batch_ext_clear(self.batch_ext.as_ptr()) }; - - for entry in batch.entries() { - let mut out_llama_cpp_return_code: i32 = 0; - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_batch_ext_add_entry( - self.batch_ext.as_ptr(), - entry.token.0, - entry.pos, - entry.seq_ids.as_ptr(), - entry.seq_ids.len(), - entry.output, - &raw mut out_llama_cpp_return_code, - &raw mut out_error, - ) - }; - - batch_ext_add_entry_status_to_result(status, out_error)?; - } + let mut out_llama_cpp_return_code: i32 = 0; + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_batch_ext_assign_entries( + self.batch_ext.as_ptr(), + batch.entries().as_ptr(), + batch.entries().len(), + batch.seq_ids().as_ptr(), + batch.seq_ids().len(), + &raw mut out_llama_cpp_return_code, + &raw mut out_error, + ) + }; - Ok(()) + batch_ext_assign_entries_status_to_result(status, out_error) } fn process( @@ -619,9 +630,7 @@ impl<'model> LlamaContext<'model> { }; process_status_to_result::(status, out_llama_cpp_return_code, out_error)?; - self.record_decoded_rows(DecodedOutputRows::BatchIndices( - batch.output_token_indices().to_vec(), - )); + self.record_batch_output_rows(batch.output_token_indices()); Ok(()) } @@ -630,7 +639,7 @@ impl<'model> LlamaContext<'model> { /// /// - `DecodeError` if the decoding failed. pub fn decode(&mut self, batch: &LlamaBatch) -> Result<(), DecodeError> { - if self.decodes_batches_in_one_micro_batch() && self.exceeds_micro_batch(batch) { + if self.decodes_batches_in_one_micro_batch && self.exceeds_micro_batch(batch) { return Err(DecodeError::BatchExceedsMicroBatch { n_tokens: batch.n_tokens(), n_ubatch: self.n_ubatch(), @@ -660,10 +669,8 @@ impl<'model> LlamaContext<'model> { } #[must_use] - pub fn pooling_type(&self) -> LlamaPoolingType { - LlamaPoolingType::from(unsafe { - llama_cpp_bindings_sys::llama_pooling_type(self.context.as_ptr()) - }) + pub const fn pooling_type(&self) -> LlamaPoolingType { + self.decoded_outputs.pooling_type } pub fn set_embeddings(&mut self, embeddings: bool) { @@ -1413,64 +1420,72 @@ mod batch_ext_init_status_to_result_tests { } #[cfg(test)] -mod batch_ext_add_entry_status_to_result_tests { +mod batch_ext_assign_entries_status_to_result_tests { use std::ptr; - use super::batch_ext_add_entry_status_to_result; + use super::batch_ext_assign_entries_status_to_result; use crate::error::batch_transfer_error::BatchTransferError; #[test] fn adding_a_batch_entry_reports_each_refusal_of_a_validated_entry_as_a_contract_violation() { for (status, detail) in [ ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_BATCH_FULL, + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_BATCH_FULL, "refused an entry of a batch validated against its limits as full", ), ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_INVALID_TOKEN, + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_TOKEN, "refused a token validated against the vocabulary", ), ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_INVALID_SEQUENCE_ID, + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_SEQUENCE_ID, "refused a sequence id validated against the context", ), ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_RETURNED_UNKNOWN_CODE, + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_RETURNED_UNKNOWN_CODE, "refused an entry with an undocumented code", ), ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_POSITION_REJECTED, + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_POSITION_REJECTED, "refused the position of an entry it had just added", ), ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_OUTPUT_REJECTED, + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OUTPUT_REJECTED, "refused the output flag of an entry it had just added", ), ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_EMPTY_SEQ_IDS, + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_EMPTY_SEQ_IDS, "was given an entry without sequence ids", ), ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_BATCH_ARG, + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_SEQ_IDS_OUT_OF_RANGE, + "was given an entry whose sequence ids lie outside the sequence id buffer", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_ENTRIES_ARG, + "was given a null entries argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_BATCH_ARG, "was given a null batch argument", ), ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_SEQ_IDS_ARG, + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_SEQ_IDS_ARG, "was given a null seq_ids argument", ), ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG, + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG, "was given a null out_llama_cpp_return_code argument", ), ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_NULL_OUT_ERROR_ARG, + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_ERROR_ARG, "was given a null out_error argument", ), ] { assert_eq!( - batch_ext_add_entry_status_to_result(status, ptr::null_mut()), + batch_ext_assign_entries_status_to_result(status, ptr::null_mut()), Err(BatchTransferError::FfiContract(llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_batch_ext_add_entry", + operation: "llama_rs_batch_ext_assign_entries", detail, })) ); @@ -1481,10 +1496,10 @@ mod batch_ext_add_entry_status_to_result_tests { fn adding_a_batch_entry_reports_memory_exhaustion() { assert_eq!( [ - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_ERROR_STRING_ALLOCATION_FAILED, - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_OUT_OF_MEMORY, + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_ERROR_STRING_ALLOCATION_FAILED, + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_OUT_OF_MEMORY, ] - .map(|status| batch_ext_add_entry_status_to_result(status, ptr::null_mut())), + .map(|status| batch_ext_assign_entries_status_to_result(status, ptr::null_mut())), [ Err(BatchTransferError::NotEnoughMemory), Err(BatchTransferError::LlamaCppOutOfMemory), @@ -1499,8 +1514,8 @@ mod batch_ext_add_entry_status_to_result_tests { }; assert_eq!( - batch_ext_add_entry_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_THREW_CXX_EXCEPTION, + batch_ext_assign_entries_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_THREW_CXX_EXCEPTION, out_error, ), Err(BatchTransferError::Reported { @@ -1512,13 +1527,13 @@ mod batch_ext_add_entry_status_to_result_tests { #[test] fn adding_a_batch_entry_reports_an_exception_without_a_message_as_a_contract_violation() { assert_eq!( - batch_ext_add_entry_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ADD_ENTRY_LLAMA_CPP_THREW_CXX_EXCEPTION, + batch_ext_assign_entries_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), ), Err(BatchTransferError::FfiContract( llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_batch_ext_add_entry", + operation: "llama_rs_batch_ext_assign_entries", detail: "reported a thrown C++ exception without an error message", } )) @@ -1528,10 +1543,10 @@ mod batch_ext_add_entry_status_to_result_tests { #[test] fn adding_a_batch_entry_preserves_an_unknown_status() { assert_eq!( - batch_ext_add_entry_status_to_result(255, ptr::null_mut()), + batch_ext_assign_entries_status_to_result(255, ptr::null_mut()), Err(BatchTransferError::FfiStatus( llama_cpp_ffi_status::FfiStatusError { - operation: "llama_rs_batch_ext_add_entry", + operation: "llama_rs_batch_ext_assign_entries", code: 255, } )) diff --git a/llama-cpp-bindings/src/context/context_limits.rs b/llama-cpp-bindings/src/context/context_limits.rs index adf7d0cb2..95ec33cff 100644 --- a/llama-cpp-bindings/src/context/context_limits.rs +++ b/llama-cpp-bindings/src/context/context_limits.rs @@ -17,13 +17,8 @@ pub struct ContextLimits { impl ContextLimits { #[must_use] - pub fn of(context: &LlamaContext) -> Self { - Self { - n_batch: context.n_batch(), - n_ubatch: context.n_ubatch(), - n_seq_max: context.n_seq_max(), - n_vocab: context.model.n_vocab(), - } + pub const fn of(context: &LlamaContext) -> Self { + context.limits } /// # Errors diff --git a/llama-cpp-bindings/src/context/decoded_outputs.rs b/llama-cpp-bindings/src/context/decoded_outputs.rs index b2b75d3ca..f701e3321 100644 --- a/llama-cpp-bindings/src/context/decoded_outputs.rs +++ b/llama-cpp-bindings/src/context/decoded_outputs.rs @@ -19,7 +19,7 @@ impl DecodedOutputs { pub fn ensure_output(&self, token_index: i32) -> Result<(), LogitsError> { let names_an_output = match &self.rows { DecodedOutputRows::BatchIndices(output_token_indices) if token_index >= 0 => { - output_token_indices.contains(&token_index) + output_token_indices.binary_search(&token_index).is_ok() } DecodedOutputRows::BatchIndices(output_token_indices) => { token_index.unsigned_abs() as usize <= output_token_indices.len() diff --git a/llama-cpp-bindings/src/lib.rs b/llama-cpp-bindings/src/lib.rs index cc8dc5e3b..8516d2f25 100644 --- a/llama-cpp-bindings/src/lib.rs +++ b/llama-cpp-bindings/src/lib.rs @@ -47,7 +47,6 @@ pub mod llama_backend_device; pub mod llama_backend_device_type; pub mod llama_backend_numa_strategy; pub mod llama_batch; -pub mod llama_batch_entry; pub mod llama_time_us; pub mod llama_token_attr; pub mod llama_token_attrs; diff --git a/llama-cpp-bindings/src/llama_batch.rs b/llama-cpp-bindings/src/llama_batch.rs index f7930e4cd..d76b24f14 100644 --- a/llama-cpp-bindings/src/llama_batch.rs +++ b/llama-cpp-bindings/src/llama_batch.rs @@ -1,15 +1,17 @@ -use llama_cpp_bindings_sys::{llama_pos, llama_seq_id}; +use llama_cpp_bindings_sys::{llama_pos, llama_rs_batch_entry, llama_seq_id}; use crate::batch_add_error::BatchAddError; use crate::context::context_limits::ContextLimits; -use crate::llama_batch_entry::LlamaBatchEntry; use crate::sampled_token::SampledToken; use crate::token::LlamaToken; +/// Tokens to decode together. Each entry names its sequences by a range of `seq_ids`, the layout +/// llama.cpp receives the whole batch in, in one call. #[derive(Clone, Debug, Eq, PartialEq)] pub struct LlamaBatch { limits: ContextLimits, - entries: Vec, + entries: Vec, + seq_ids: Vec, output_token_indices: Vec, n_tokens: i32, } @@ -20,6 +22,7 @@ impl LlamaBatch { Self { limits, entries: Vec::new(), + seq_ids: Vec::new(), output_token_indices: Vec::new(), n_tokens: 0, } @@ -27,6 +30,7 @@ impl LlamaBatch { pub fn clear(&mut self) { self.entries.clear(); + self.seq_ids.clear(); self.output_token_indices.clear(); self.n_tokens = 0; } @@ -72,12 +76,14 @@ impl LlamaBatch { self.output_token_indices.push(token_index); } - self.entries.push(LlamaBatchEntry { - token, + self.entries.push(llama_rs_batch_entry { + token: token.0, pos, - seq_ids: seq_ids.to_vec(), + seq_ids_offset: self.seq_ids.len(), + seq_ids_count: seq_ids.len(), output: logits, }); + self.seq_ids.extend_from_slice(seq_ids); Ok(()) } @@ -117,10 +123,15 @@ impl LlamaBatch { } #[must_use] - pub fn entries(&self) -> &[LlamaBatchEntry] { + pub fn entries(&self) -> &[llama_rs_batch_entry] { &self.entries } + #[must_use] + pub fn seq_ids(&self) -> &[llama_seq_id] { + &self.seq_ids + } + #[must_use] pub fn output_token_indices(&self) -> &[i32] { &self.output_token_indices @@ -129,11 +140,12 @@ impl LlamaBatch { #[cfg(test)] mod tests { + use llama_cpp_bindings_sys::llama_rs_batch_entry; + use crate::batch_add_error::BatchAddError; use crate::context::context_limits::ContextLimits; use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; use crate::error::token_outside_vocabulary::TokenOutsideVocabulary; - use crate::llama_batch_entry::LlamaBatchEntry; use crate::sampled_token::SampledToken; use crate::token::LlamaToken; @@ -176,20 +188,23 @@ mod tests { assert_eq!( batch.entries(), [ - LlamaBatchEntry { - token: LlamaToken::new(1), + llama_rs_batch_entry { + token: 1, pos: 0, - seq_ids: vec![0], + seq_ids_offset: 0, + seq_ids_count: 1, output: false, }, - LlamaBatchEntry { - token: LlamaToken::new(2), + llama_rs_batch_entry { + token: 2, pos: 1, - seq_ids: vec![0, 1], + seq_ids_offset: 1, + seq_ids_count: 2, output: true, }, ] ); + assert_eq!(batch.seq_ids(), [0, 0, 1]); assert_eq!(batch.output_token_indices(), [1]); assert_eq!(batch.n_tokens(), 2); } @@ -267,10 +282,11 @@ mod tests { assert_eq!(batch.output_token_indices(), [2]); assert_eq!( batch.entries().last(), - Some(&LlamaBatchEntry { - token: LlamaToken::new(30), + Some(&llama_rs_batch_entry { + token: 30, pos: 2, - seq_ids: vec![1], + seq_ids_offset: 2, + seq_ids_count: 1, output: true, }) ); diff --git a/llama-cpp-bindings/src/llama_batch_entry.rs b/llama-cpp-bindings/src/llama_batch_entry.rs deleted file mode 100644 index 1a73063e8..000000000 --- a/llama-cpp-bindings/src/llama_batch_entry.rs +++ /dev/null @@ -1,11 +0,0 @@ -use llama_cpp_bindings_sys::{llama_pos, llama_seq_id}; - -use crate::token::LlamaToken; - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct LlamaBatchEntry { - pub token: LlamaToken, - pub pos: llama_pos, - pub seq_ids: Vec, - pub output: bool, -} diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index 6fde4fe1e..305a9e1a1 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -24,6 +24,7 @@ use std::ptr; use std::ptr::NonNull; use std::sync::Arc; +use encoding_rs::CoderResult; use once_cell::sync::OnceCell; use toktrie::ApproximateTokEnv; use toktrie::INVALID_TOKEN; @@ -56,7 +57,9 @@ use crate::model::vocab_type_from_int_error::VocabTypeFromIntError; use llama_cpp_ffi_status::read_and_free_cpp_string; use params::LlamaModelParams; -const PIECE_PROBE_SIZE: usize = 32; +const PIECE_PROBE_LEN: c_int = 32; + +const PIECE_PROBE_SIZE: usize = PIECE_PROBE_LEN.unsigned_abs() as usize; fn validate_string_length_for_tokenizer(length: usize) -> Result { Ok(c_int::try_from(length)?) @@ -82,8 +85,27 @@ const fn present_special_token( } } +fn decode_piece(decoder: &mut encoding_rs::Decoder, bytes: &[u8]) -> String { + let mut piece = String::with_capacity(bytes.len()); + let mut undecoded = bytes; + + loop { + let (result, read, _had_replacements) = + decoder.decode_to_string(undecoded, &mut piece, false); + + undecoded = undecoded.split_at(read).1; + + match result { + CoderResult::InputEmpty => return piece, + CoderResult::OutputFull => piece.reserve(char::MAX_LEN_UTF8), + } + } +} + pub struct LlamaModel { pub model: NonNull, + vocab: *const llama_cpp_bindings_sys::llama_vocab, + n_vocab: i32, tok_env: OnceCell>, chat_templates: OnceCell, } @@ -119,8 +141,12 @@ unsafe fn load_model_from_file_status_to_result( detail: "success status contained a null model", }) })?; + let vocab = unsafe { llama_cpp_bindings_sys::llama_model_get_vocab(model.as_ptr()) }; + Ok(LlamaModel { model, + vocab, + n_vocab: unsafe { llama_cpp_bindings_sys::llama_vocab_n_tokens(vocab) }, tok_env: OnceCell::new(), chat_templates: OnceCell::new(), }) @@ -186,8 +212,8 @@ impl LlamaModel { } #[must_use] - pub fn vocab_ptr(&self) -> *const llama_cpp_bindings_sys::llama_vocab { - unsafe { llama_cpp_bindings_sys::llama_model_get_vocab(self.model.as_ptr()) } + pub const fn vocab_ptr(&self) -> *const llama_cpp_bindings_sys::llama_vocab { + self.vocab } /// # Errors @@ -330,13 +356,23 @@ impl LlamaModel { | SampledToken::Reasoning(inner) | SampledToken::ToolCall(inner) | SampledToken::Undeterminable(inner)) = *token; - let bytes = self.token_to_piece_bytes_fitted(inner, special, lstrip)?; + let token = self.require_vocabulary_token(inner)?; + let mut probe = [0_u8; PIECE_PROBE_SIZE]; - let mut output_piece = String::with_capacity(bytes.len()); - let (_result, _decoded_size, _had_replacements) = - decoder.decode_to_string(&bytes, &mut output_piece, false); + match self.write_piece(token, &mut probe, PIECE_PROBE_LEN, special, lstrip) { + Ok(written) => Ok(decode_piece(decoder, probe.split_at(written).0)), + Err(TokenToStringError::InsufficientBufferSpace(required_size)) => { + let bytes = self.token_to_piece_bytes( + token, + required_size.unsigned_abs() as usize, + special, + lstrip, + )?; - Ok(output_piece) + Ok(decode_piece(decoder, &bytes)) + } + Err(probe_error) => Err(probe_error), + } } /// # Errors @@ -355,14 +391,28 @@ impl LlamaModel { let token = self.require_vocabulary_token(token)?; let buffer_len = c_int::try_from(buffer_size)?; let mut buffer: Vec = vec![0u8; buffer_size]; - let lstrip = lstrip.map_or(0, |strip_count| i32::from(strip_count.get())); + let written = self.write_piece(token, &mut buffer, buffer_len, special, lstrip)?; + + buffer.truncate(written); + + Ok(buffer) + } + + fn write_piece( + &self, + token: LlamaToken, + buffer: &mut [u8], + buffer_len: c_int, + special: bool, + lstrip: Option, + ) -> Result { let size = unsafe { llama_cpp_bindings_sys::llama_rs_token_to_piece( - self.vocab_ptr(), + self.vocab, token.0, buffer.as_mut_ptr().cast::(), buffer_len, - lstrip, + lstrip.map_or(0, |strip_count| i32::from(strip_count.get())), special, ) }; @@ -372,11 +422,7 @@ impl LlamaModel { error_code if error_code.is_negative() => { Err(TokenToStringError::InsufficientBufferSpace(error_code)) } - size => { - buffer.truncate(size.unsigned_abs() as usize); - - Ok(buffer) - } + size => Ok(size.unsigned_abs() as usize), } } @@ -399,8 +445,8 @@ impl LlamaModel { } #[must_use] - pub fn n_vocab(&self) -> i32 { - unsafe { llama_cpp_bindings_sys::llama_vocab_n_tokens(self.vocab_ptr()) } + pub const fn n_vocab(&self) -> i32 { + self.n_vocab } /// # Errors diff --git a/llama-cpp-bindings/src/mtmd/mtmd_eval_limits.rs b/llama-cpp-bindings/src/mtmd/mtmd_eval_limits.rs index d7dddfb9a..5b88c4123 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_eval_limits.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_eval_limits.rs @@ -12,7 +12,7 @@ pub struct MtmdEvalLimits { impl MtmdEvalLimits { #[must_use] - pub fn of(llama_ctx: &LlamaContext) -> Self { + pub const fn of(llama_ctx: &LlamaContext) -> Self { Self { context: ContextLimits::of(llama_ctx), } From 36da36e1f6891d53b41c8f3bd5aba965cfe8f63b Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sat, 31 Oct 2026 05:16:43 +0100 Subject: [PATCH 25/82] Import inline paths at module scope and order imports as std, vendor, project and local crates --- llama-cpp-bindings-build/src/android_ndk.rs | 36 ++--- .../src/bindings_generation.rs | 2 + .../src/link_directives.rs | 17 ++- llama-cpp-bindings-build/src/native_build.rs | 17 ++- .../src/rebuild_triggers.rs | 1 + .../src/wrapper_compilation.rs | 1 + llama-cpp-bindings-sys/build.rs | 4 +- .../src/media_question_chat_session.rs | 1 + .../src/paris_weather_tool_call.rs | 3 +- llama-cpp-bindings-tests/src/prime_memory.rs | 1 + .../src/prime_memory_with.rs | 1 + .../src/user_turn_chat_session.rs | 1 + .../approximate_tok_env_eos_selection.rs | 15 +- .../tests/batch_context_limits.rs | 1 + .../tests/chat_answer_parsing.rs | 1 + .../tests/chat_prompt_rendering.rs | 1 + .../tests/chat_session_input_validation.rs | 1 + .../tests/chat_template_tool_call_support.rs | 1 + .../tests/completion_usage_accounting.rs | 1 + .../tests/context_dimensions.rs | 1 + .../tests/context_synchronization.rs | 1 + .../tests/debug_formatting.rs | 1 + .../tests/decode_abort_control.rs | 1 + .../tests/decode_batch_rejection.rs | 1 + .../tests/detokenization.rs | 1 + .../tests/duck_typed_tool_calls.rs | 1 + .../tests/embedding_availability.rs | 3 +- .../tests/embedding_generation.rs | 1 + .../tests/embedding_retrieval_errors.rs | 1 + .../tests/embedding_row_width.rs | 4 +- .../tests/encode_batch_rejection.rs | 1 + .../tests/encoder_requirement.rs | 1 + .../end_of_generation_token_detection.rs | 1 + .../tests/generation_end_classification.rs | 1 + .../tests/grammar_sampler.rs | 1 + .../tests/hidden_state_extraction.rs | 1 + .../tests/image_question_answering.rs | 1 + .../json_schema_constrained_generation.rs | 1 + .../tests/lazy_grammar_sampler.rs | 1 + .../tests/llguidance_sampler.rs | 1 + .../tests/logits_availability.rs | 4 +- .../tests/logits_retrieval.rs | 1 + .../tests/lora_adapters.rs | 1 + .../tests/memory_availability.rs | 1 + .../tests/memory_occupancy.rs | 1 + .../tests/memory_position_bounds.rs | 1 + ...emory_position_shift_rope_compatibility.rs | 1 + .../tests/memory_priming.rs | 1 + .../tests/memory_sequence_edits.rs | 1 + .../tests/memory_sequence_id_bounds.rs | 4 +- .../tests/model_architecture.rs | 4 +- .../tests/model_derived_data_caching.rs | 1 + .../tests/model_hyperparameters.rs | 1 + .../tests/model_loading_errors.rs | 23 ++- .../tests/model_metadata.rs | 1 + .../tests/multimodal_audio.rs | 1 + .../tests/multimodal_bitmap_construction.rs | 5 +- .../tests/multimodal_chunk_copy.rs | 1 + .../tests/multimodal_chunk_encoding.rs | 1 + .../tests/multimodal_chunk_evaluation.rs | 4 +- .../tests/multimodal_chunk_tokenization.rs | 4 +- .../tests/multimodal_fusion.rs | 1 + .../multimodal_projector_capabilities.rs | 1 + .../multimodal_prompt_parts_tokenization.rs | 1 + .../multimodal_reasoning_classification.rs | 1 + .../tests/multimodal_usage_accounting.rs | 1 + .../tests/parameter_fitting.rs | 1 + .../tests/prompt_token_accounting.rs | 1 + .../tests/reasoning_classification.rs | 1 + llama-cpp-bindings-tests/tests/reranking.rs | 3 +- .../tests/sampled_token_ingestion.rs | 1 + .../tests/sampler_token_acceptance.rs | 1 + .../tests/session_file_path_validation.rs | 33 ++--- .../tests/session_file_token_capacity.rs | 20 ++- .../tests/session_files.rs | 12 +- .../tests/special_tokens.rs | 1 + .../tests/state_snapshots.rs | 15 +- .../tests/threadpool_detachment.rs | 1 + .../tests/token_attributes.rs | 1 + .../tests/token_candidates.rs | 1 + .../tests/tokenization.rs | 5 +- .../tests/tool_call_classification.rs | 1 + .../tests/tool_call_grammar.rs | 1 + .../tests/tool_call_parsing.rs | 3 +- .../tests/unconstrained_token_sampling.rs | 1 + .../tests/vocabulary_enumeration.rs | 1 + .../tests/vocabulary_type.rs | 4 +- llama-cpp-bindings-types/src/token_usage.rs | 4 +- llama-cpp-bindings/build.rs | 4 +- .../src/chat_message_parse_outcome.rs | 3 +- llama-cpp-bindings/src/chat_session.rs | 5 +- llama-cpp-bindings/src/chat_spans.rs | 3 +- llama-cpp-bindings/src/context.rs | 41 +++--- .../src/context/llama_state_seq_flags.rs | 4 +- llama-cpp-bindings/src/context/memory.rs | 3 +- llama-cpp-bindings/src/context/params.rs | 32 ++-- llama-cpp-bindings/src/gguf_context.rs | 138 +++++++++--------- llama-cpp-bindings/src/grammar_matcher.rs | 11 +- .../src/json_schema_to_grammar.rs | 33 ++--- llama-cpp-bindings/src/llama_backend.rs | 22 +-- .../src/llama_backend_device.rs | 19 ++- llama-cpp-bindings/src/llama_batch.rs | 3 +- llama-cpp-bindings/src/llguidance_sampler.rs | 25 ++-- .../src/load_backends_from_path.rs | 24 +-- llama-cpp-bindings/src/model.rs | 95 ++++++------ .../src/model/llama_chat_template.rs | 10 +- .../src/model/llama_load_mode.rs | 7 +- .../src/model/llama_lora_adapter.rs | 3 +- llama-cpp-bindings/src/model/params.rs | 112 ++++++-------- .../src/model/params/kv_overrides.rs | 8 +- .../src/model/params/param_override_value.rs | 8 +- llama-cpp-bindings/src/mtmd/eval_chunk.rs | 8 +- .../src/mtmd/fitted_mtmd_input_chunk.rs | 3 +- llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs | 26 ++-- llama-cpp-bindings/src/mtmd/mtmd_context.rs | 4 +- .../src/mtmd/mtmd_eval_limits.rs | 3 +- .../src/mtmd/mtmd_input_chunk.rs | 7 +- .../src/mtmd/mtmd_input_chunks.rs | 3 +- .../src/mtmd/mtmd_memory_usage.rs | 3 +- .../src/sampled_token_classifier.rs | 1 - llama-cpp-bindings/src/sampling.rs | 123 ++++++++-------- llama-cpp-bindings/src/send_logs_to_log.rs | 51 ++++--- llama-cpp-bindings/src/token.rs | 4 +- llama-cpp-bindings/src/token/data_array.rs | 3 +- llama-cpp-bindings/src/tool_call_format.rs | 22 +-- .../src/tool_call_format/bracketed_args.rs | 9 +- .../src/tool_call_format/json_object.rs | 3 +- .../tool_call_format/key_value_xml_tags.rs | 10 +- .../src/tool_call_format/paired_quote_args.rs | 3 +- .../src/tool_call_format/xml_function_tags.rs | 10 +- .../tests/sampler_chain_without_backend.rs | 1 + llama-cpp-gbnf/src/validate_gbnf.rs | 24 +-- .../src/parsed_args.rs | 6 +- .../src/parsed_source.rs | 4 +- llama-cpp-test-harness/src/download_model.rs | 3 +- llama-cpp-test-harness/src/execution_phase.rs | 3 +- llama-cpp-test-harness/src/execution_plan.rs | 4 +- .../src/harness_run_error.rs | 3 +- llama-cpp-test-harness/src/lib.rs | 11 +- llama-cpp-test-harness/src/llama_fixture.rs | 1 + llama-cpp-test-harness/src/load_key.rs | 3 +- .../src/parse_harness_arguments.rs | 3 +- .../src/phase_model_files.rs | 4 +- llama-cpp-test-harness/src/phase_setup.rs | 4 +- llama-cpp-test-harness/src/run.rs | 11 +- .../src/run_to_conclusions.rs | 8 +- llama-cpp-test-harness/src/scheduled_phase.rs | 5 +- .../src/test_backend_gate.rs | 4 +- .../tests/harness_self_test.rs | 1 + .../tests/emit_compile_commands.rs | 3 +- 150 files changed, 698 insertions(+), 628 deletions(-) diff --git a/llama-cpp-bindings-build/src/android_ndk.rs b/llama-cpp-bindings-build/src/android_ndk.rs index c8083addd..e62e6197e 100644 --- a/llama-cpp-bindings-build/src/android_ndk.rs +++ b/llama-cpp-bindings-build/src/android_ndk.rs @@ -176,10 +176,13 @@ impl AndroidNdk { #[cfg(test)] pub mod tests { + use std::env; use std::ffi::OsString; + use std::fs; use std::os::unix::ffi::OsStringExt; use std::path::Path; use std::path::PathBuf; + use std::process; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; @@ -205,10 +208,7 @@ pub mod tests { fn temporary_directory(name: &str) -> PathBuf { let id = NEXT_DIRECTORY_ID.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir().join(format!( - "llama-cpp-bindings-{name}-{}-{id}", - std::process::id() - )) + env::temp_dir().join(format!("llama-cpp-bindings-{name}-{}-{id}", process::id())) } fn ndk_at(root: &Path) -> BuildEnvironment { @@ -352,7 +352,7 @@ pub mod tests { #[test] fn ndk_validation_rejects_a_root_without_a_cmake_toolchain() { let root = temporary_directory("ndk-without-toolchain"); - std::fs::create_dir_all(&root).unwrap(); + fs::create_dir_all(&root).unwrap(); assert_eq!( validate_ndk_installation(root.to_str().unwrap()) @@ -363,16 +363,16 @@ pub mod tests { root.join("build/cmake/android.toolchain.cmake").display() )) ); - std::fs::remove_dir_all(root).unwrap(); + fs::remove_dir_all(root).unwrap(); } #[test] fn detection_rejects_a_toolchain_without_a_clang_directory() { let root = temporary_directory("ndk-without-clang"); let toolchain = root.join("toolchains/llvm/prebuilt/linux-x86_64"); - std::fs::create_dir_all(root.join("build/cmake")).unwrap(); - std::fs::write(root.join("build/cmake/android.toolchain.cmake"), "").unwrap(); - std::fs::create_dir_all(&toolchain).unwrap(); + fs::create_dir_all(root.join("build/cmake")).unwrap(); + fs::write(root.join("build/cmake/android.toolchain.cmake"), "").unwrap(); + fs::create_dir_all(&toolchain).unwrap(); assert_eq!( detection_failure(&ndk_at(&root), "aarch64", HostOs::Linux), @@ -382,14 +382,14 @@ pub mod tests { toolchain.join("lib/clang").display() )) ); - std::fs::remove_dir_all(root).unwrap(); + fs::remove_dir_all(root).unwrap(); } #[test] fn clang_builtin_include_resolution_requires_a_version_directory() { let toolchain = temporary_directory("clang-without-version"); - std::fs::create_dir_all(toolchain.join("lib/clang")).unwrap(); - std::fs::write(toolchain.join("lib/clang/21"), "").unwrap(); + fs::create_dir_all(toolchain.join("lib/clang")).unwrap(); + fs::write(toolchain.join("lib/clang/21"), "").unwrap(); assert_eq!( find_clang_builtin_includes(toolchain.to_str().unwrap()) @@ -400,14 +400,14 @@ pub mod tests { toolchain.join("lib/clang").display() )) ); - std::fs::remove_dir_all(toolchain).unwrap(); + fs::remove_dir_all(toolchain).unwrap(); } #[test] fn clang_builtin_include_resolution_refuses_to_choose_between_version_directories() { let toolchain = temporary_directory("clang-with-two-versions"); - std::fs::create_dir_all(toolchain.join("lib/clang/18/include")).unwrap(); - std::fs::create_dir_all(toolchain.join("lib/clang/17/include")).unwrap(); + fs::create_dir_all(toolchain.join("lib/clang/18/include")).unwrap(); + fs::create_dir_all(toolchain.join("lib/clang/17/include")).unwrap(); assert_eq!( find_clang_builtin_includes(toolchain.to_str().unwrap()) @@ -422,13 +422,13 @@ pub mod tests { ] )) ); - std::fs::remove_dir_all(toolchain).unwrap(); + fs::remove_dir_all(toolchain).unwrap(); } #[test] fn clang_builtin_include_resolution_requires_an_include_directory() { let toolchain = temporary_directory("clang-without-includes"); - std::fs::create_dir_all(toolchain.join("lib/clang/20")).unwrap(); + fs::create_dir_all(toolchain.join("lib/clang/20")).unwrap(); assert_eq!( find_clang_builtin_includes(toolchain.to_str().unwrap()) @@ -439,6 +439,6 @@ pub mod tests { toolchain.join("lib/clang/20/include").display() )) ); - std::fs::remove_dir_all(toolchain).unwrap(); + fs::remove_dir_all(toolchain).unwrap(); } } diff --git a/llama-cpp-bindings-build/src/bindings_generation.rs b/llama-cpp-bindings-build/src/bindings_generation.rs index 6706ca65c..3e77ce1e3 100644 --- a/llama-cpp-bindings-build/src/bindings_generation.rs +++ b/llama-cpp-bindings-build/src/bindings_generation.rs @@ -1,6 +1,7 @@ use std::path::PathBuf; use bindgen::Builder; + use llama_cpp_wrapper_sources::fault_injection_define::FAULT_INJECTION_DEFINE; use crate::binding_callbacks::BindingCallbacks; @@ -194,6 +195,7 @@ mod tests { use std::path::PathBuf; use enumflags2::BitFlags; + use llama_cpp_wrapper_sources::fault_injection_define::FAULT_INJECTION_DEFINE; use super::BindingsGeneration; diff --git a/llama-cpp-bindings-build/src/link_directives.rs b/llama-cpp-bindings-build/src/link_directives.rs index c9f9362c2..ca5ca2ee8 100644 --- a/llama-cpp-bindings-build/src/link_directives.rs +++ b/llama-cpp-bindings-build/src/link_directives.rs @@ -259,8 +259,11 @@ pub fn link_directives( #[cfg(test)] mod tests { + use std::env; + use std::fs; use std::path::Path; use std::path::PathBuf; + use std::process; use enumflags2::BitFlags; @@ -411,11 +414,9 @@ mod tests { #[test] fn static_gpu_backends_link_their_archives_and_the_vendor_runtimes() { - let rocm_root = std::env::temp_dir().join(format!( - "llama-cpp-bindings-build-rocm-{}", - std::process::id() - )); - std::fs::create_dir_all(rocm_root.join("lib")).unwrap(); + let rocm_root = + env::temp_dir().join(format!("llama-cpp-bindings-build-rocm-{}", process::id())); + fs::create_dir_all(rocm_root.join("lib")).unwrap(); let environment = BuildEnvironment { cuda_library_dirs: vec![PathBuf::from("/cuda/lib64")], ..cargo_environment( @@ -472,7 +473,7 @@ mod tests { ] .concat() ); - std::fs::remove_dir_all(rocm_root).unwrap(); + fs::remove_dir_all(rocm_root).unwrap(); } #[test] @@ -613,9 +614,9 @@ mod tests { #[test] fn library_directories_an_enabled_feature_needs_must_exist() { - let missing_directory = std::env::temp_dir().join(format!( + let missing_directory = env::temp_dir().join(format!( "llama-cpp-bindings-build-missing-{}", - std::process::id() + process::id() )); let refusal = |features: BitFlags, overrides: &[(&str, &str)]| { let environment = cargo_environment(features, overrides); diff --git a/llama-cpp-bindings-build/src/native_build.rs b/llama-cpp-bindings-build/src/native_build.rs index 176495fcf..25e1dd127 100644 --- a/llama-cpp-bindings-build/src/native_build.rs +++ b/llama-cpp-bindings-build/src/native_build.rs @@ -80,8 +80,11 @@ impl NativeBuild { #[cfg(test)] mod tests { + use std::env; + use std::fs; use std::path::Path; use std::path::PathBuf; + use std::process; use super::NativeBuild; use crate::bindings_generation::BindingsGeneration; @@ -148,12 +151,10 @@ mod tests { } fn build_root(name: &str) -> PathBuf { - let root = std::env::temp_dir().join(format!( - "llama-cpp-bindings-build-{name}-{}", - std::process::id() - )); + let root = + env::temp_dir().join(format!("llama-cpp-bindings-build-{name}-{}", process::id())); - std::fs::create_dir_all(&root).unwrap(); + fs::create_dir_all(&root).unwrap(); root } @@ -191,7 +192,7 @@ mod tests { ], [true; 4] ); - std::fs::remove_dir_all(root).unwrap(); + fs::remove_dir_all(root).unwrap(); } } @@ -216,7 +217,7 @@ mod tests { [output.is_empty(), root.join("cmake").exists()], [true, false] ); - std::fs::remove_dir_all(root).unwrap(); + fs::remove_dir_all(root).unwrap(); } #[test] @@ -238,6 +239,6 @@ mod tests { )) ); assert_eq!(output, Vec::::new()); - std::fs::remove_dir_all(root).unwrap(); + fs::remove_dir_all(root).unwrap(); } } diff --git a/llama-cpp-bindings-build/src/rebuild_triggers.rs b/llama-cpp-bindings-build/src/rebuild_triggers.rs index 62c89b76c..1617ed21f 100644 --- a/llama-cpp-bindings-build/src/rebuild_triggers.rs +++ b/llama-cpp-bindings-build/src/rebuild_triggers.rs @@ -27,6 +27,7 @@ mod tests { use std::path::PathBuf; use enumflags2::BitFlags; + use llama_cpp_wrapper_sources::wrapper_headers::WRAPPER_HEADERS; use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; diff --git a/llama-cpp-bindings-build/src/wrapper_compilation.rs b/llama-cpp-bindings-build/src/wrapper_compilation.rs index 3e3355fd3..0c332653d 100644 --- a/llama-cpp-bindings-build/src/wrapper_compilation.rs +++ b/llama-cpp-bindings-build/src/wrapper_compilation.rs @@ -100,6 +100,7 @@ mod tests { use std::path::PathBuf; use enumflags2::BitFlags; + use llama_cpp_wrapper_sources::fault_injection_define::FAULT_INJECTION_DEFINE; use llama_cpp_wrapper_sources::system_include_flags::system_include_flags; use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; diff --git a/llama-cpp-bindings-sys/build.rs b/llama-cpp-bindings-sys/build.rs index 4d04a0415..bd7ad472f 100644 --- a/llama-cpp-bindings-sys/build.rs +++ b/llama-cpp-bindings-sys/build.rs @@ -1,6 +1,8 @@ +use std::io; + use llama_cpp_bindings_build::build_environment::BuildEnvironment; use llama_cpp_bindings_build::build_error::BuildError; fn main() -> Result<(), BuildError> { - llama_cpp_bindings_build::build(&BuildEnvironment::of_process(), &mut std::io::stdout()) + llama_cpp_bindings_build::build(&BuildEnvironment::of_process(), &mut io::stdout()) } diff --git a/llama-cpp-bindings-tests/src/media_question_chat_session.rs b/llama-cpp-bindings-tests/src/media_question_chat_session.rs index 25db5413a..1798f000b 100644 --- a/llama-cpp-bindings-tests/src/media_question_chat_session.rs +++ b/llama-cpp-bindings-tests/src/media_question_chat_session.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings::model::LlamaModel; use llama_cpp_bindings::mtmd::mtmd_default_marker::mtmd_default_marker; diff --git a/llama-cpp-bindings-tests/src/paris_weather_tool_call.rs b/llama-cpp-bindings-tests/src/paris_weather_tool_call.rs index f44af1495..9b1b4e919 100644 --- a/llama-cpp-bindings-tests/src/paris_weather_tool_call.rs +++ b/llama-cpp-bindings-tests/src/paris_weather_tool_call.rs @@ -1,7 +1,8 @@ +use serde_json::json; + use llama_cpp_bindings_types::ParsedToolCall; use llama_cpp_bindings_types::ParsedToolCallId; use llama_cpp_bindings_types::ToolCallArguments; -use serde_json::json; /// The `get_weather` call for Paris that the tool-call fixtures of every format encode. #[must_use] diff --git a/llama-cpp-bindings-tests/src/prime_memory.rs b/llama-cpp-bindings-tests/src/prime_memory.rs index 4704a334e..5f1d78ac5 100644 --- a/llama-cpp-bindings-tests/src/prime_memory.rs +++ b/llama-cpp-bindings-tests/src/prime_memory.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_test_harness::LlamaFixture; diff --git a/llama-cpp-bindings-tests/src/prime_memory_with.rs b/llama-cpp-bindings-tests/src/prime_memory_with.rs index e4f03c3bf..32ed538e6 100644 --- a/llama-cpp-bindings-tests/src/prime_memory_with.rs +++ b/llama-cpp-bindings-tests/src/prime_memory_with.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; diff --git a/llama-cpp-bindings-tests/src/user_turn_chat_session.rs b/llama-cpp-bindings-tests/src/user_turn_chat_session.rs index 050806f82..14d656819 100644 --- a/llama-cpp-bindings-tests/src/user_turn_chat_session.rs +++ b/llama-cpp-bindings-tests/src/user_turn_chat_session.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings::chat_session_options::ChatSessionOptions; diff --git a/llama-cpp-bindings-tests/tests/approximate_tok_env_eos_selection.rs b/llama-cpp-bindings-tests/tests/approximate_tok_env_eos_selection.rs index 46d2ac7d0..b46dfef4c 100644 --- a/llama-cpp-bindings-tests/tests/approximate_tok_env_eos_selection.rs +++ b/llama-cpp-bindings-tests/tests/approximate_tok_env_eos_selection.rs @@ -1,13 +1,18 @@ +use std::env; +use std::fs; +use std::process; + use anyhow::Context; use anyhow::Result; +use toktrie::INVALID_TOKEN; +use toktrie::TokenizerEnv as _; + use llama_cpp_bindings::gguf_context::GgufContext; use llama_cpp_bindings::model::LlamaModel; use llama_cpp_bindings::model::params::LlamaModelParams; use llama_cpp_bindings::token::LlamaToken; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; -use toktrie::INVALID_TOKEN; -use toktrie::TokenizerEnv as _; #[llama_test( model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), @@ -46,9 +51,9 @@ fn approximate_tok_env_falls_back_to_eos_when_eot_unavailable( fn approximate_tok_env_of_a_model_without_eot_and_eos_has_no_eos( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let model_without_eos_path = std::env::temp_dir().join(format!( + let model_without_eos_path = env::temp_dir().join(format!( "llama_cpp_bindings_{}_model_without_eos.gguf", - std::process::id() + process::id() )); let mut gguf_context = GgufContext::from_file(fixture.model_path)?; @@ -61,7 +66,7 @@ fn approximate_tok_env_of_a_model_without_eot_and_eos_has_no_eos( &LlamaModelParams::default(), )?; - std::fs::remove_file(&model_without_eos_path)?; + fs::remove_file(&model_without_eos_path)?; assert_eq!( model_without_eos diff --git a/llama-cpp-bindings-tests/tests/batch_context_limits.rs b/llama-cpp-bindings-tests/tests/batch_context_limits.rs index 7ecbc4163..648a6794e 100644 --- a/llama-cpp-bindings-tests/tests/batch_context_limits.rs +++ b/llama-cpp-bindings-tests/tests/batch_context_limits.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::error::batch_transfer_error::BatchTransferError; use llama_cpp_bindings::error::decode_error::DecodeError; diff --git a/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs b/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs index 3399fd0c3..78c0bb30e 100644 --- a/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs +++ b/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs @@ -1,5 +1,6 @@ use anyhow::Result; use anyhow::bail; + use llama_cpp_bindings::chat_message_parse_outcome::ChatMessageParseOutcome; use llama_cpp_bindings::chat_output::ChatOutput; use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; diff --git a/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs b/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs index 3cec25668..6bb3da7a3 100644 --- a/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs +++ b/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings::chat_session_options::ChatSessionOptions; diff --git a/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs index f3f1c0a50..5ef552a13 100644 --- a/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs +++ b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs @@ -2,6 +2,7 @@ use std::ffi::CString; use anyhow::Result; use anyhow::bail; + use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings::chat_session_options::ChatSessionOptions; diff --git a/llama-cpp-bindings-tests/tests/chat_template_tool_call_support.rs b/llama-cpp-bindings-tests/tests/chat_template_tool_call_support.rs index 4d0fc27f3..3eea826d9 100644 --- a/llama-cpp-bindings-tests/tests/chat_template_tool_call_support.rs +++ b/llama-cpp-bindings-tests/tests/chat_template_tool_call_support.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs index b135f27c9..1f15a98a5 100644 --- a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs +++ b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs @@ -1,5 +1,6 @@ use anyhow::Context as _; use anyhow::Result; + use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; use llama_cpp_bindings::chat_session::ChatSession; diff --git a/llama-cpp-bindings-tests/tests/context_dimensions.rs b/llama-cpp-bindings-tests/tests/context_dimensions.rs index dfd672092..13b01877e 100644 --- a/llama-cpp-bindings-tests/tests/context_dimensions.rs +++ b/llama-cpp-bindings-tests/tests/context_dimensions.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/context_synchronization.rs b/llama-cpp-bindings-tests/tests/context_synchronization.rs index c12dab152..38855784e 100644 --- a/llama-cpp-bindings-tests/tests/context_synchronization.rs +++ b/llama-cpp-bindings-tests/tests/context_synchronization.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/debug_formatting.rs b/llama-cpp-bindings-tests/tests/debug_formatting.rs index a32fc2f6f..6dd78d890 100644 --- a/llama-cpp-bindings-tests/tests/debug_formatting.rs +++ b/llama-cpp-bindings-tests/tests/debug_formatting.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/decode_abort_control.rs b/llama-cpp-bindings-tests/tests/decode_abort_control.rs index 295759eda..5962a7640 100644 --- a/llama-cpp-bindings-tests/tests/decode_abort_control.rs +++ b/llama-cpp-bindings-tests/tests/decode_abort_control.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; use anyhow::Result; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::error::decode_error::DecodeError; diff --git a/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs b/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs index 19456bc3b..e413fd4fc 100644 --- a/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs +++ b/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::context::llama_attention_type::LlamaAttentionType; diff --git a/llama-cpp-bindings-tests/tests/detokenization.rs b/llama-cpp-bindings-tests/tests/detokenization.rs index 78ad55f57..a6cd0258b 100644 --- a/llama-cpp-bindings-tests/tests/detokenization.rs +++ b/llama-cpp-bindings-tests/tests/detokenization.rs @@ -2,6 +2,7 @@ use std::num::NonZeroU16; use anyhow::Context as _; use anyhow::Result; + use llama_cpp_bindings::error::token_to_string_error::TokenToStringError; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; diff --git a/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs b/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs index c32386e52..5062abd41 100644 --- a/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs +++ b/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; use llama_cpp_bindings_tests::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; use llama_cpp_bindings_tests::paris_weather_tool_call::paris_weather_tool_call; diff --git a/llama-cpp-bindings-tests/tests/embedding_availability.rs b/llama-cpp-bindings-tests/tests/embedding_availability.rs index 6c2c0b64c..b14d43c69 100644 --- a/llama-cpp-bindings-tests/tests/embedding_availability.rs +++ b/llama-cpp-bindings-tests/tests/embedding_availability.rs @@ -1,10 +1,9 @@ use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; use llama_cpp_bindings::error::embeddings_error::EmbeddingsError; +use llama_cpp_bindings_tests::context_after_decoding_text::context_after_decoding_text; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; -use llama_cpp_bindings_tests::context_after_decoding_text::context_after_decoding_text; - #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), n_gpu_layers = 999, diff --git a/llama-cpp-bindings-tests/tests/embedding_generation.rs b/llama-cpp-bindings-tests/tests/embedding_generation.rs index f8d444980..7cb2c9438 100644 --- a/llama-cpp-bindings-tests/tests/embedding_generation.rs +++ b/llama-cpp-bindings-tests/tests/embedding_generation.rs @@ -1,5 +1,6 @@ use anyhow::Context; use anyhow::Result; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; diff --git a/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs b/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs index 7c28b52be..80778ee17 100644 --- a/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs +++ b/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; diff --git a/llama-cpp-bindings-tests/tests/embedding_row_width.rs b/llama-cpp-bindings-tests/tests/embedding_row_width.rs index 10f50b7f6..d908d9f90 100644 --- a/llama-cpp-bindings-tests/tests/embedding_row_width.rs +++ b/llama-cpp-bindings-tests/tests/embedding_row_width.rs @@ -1,10 +1,10 @@ use anyhow::Result; + use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; +use llama_cpp_bindings_tests::context_after_decoding_text::context_after_decoding_text; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; -use llama_cpp_bindings_tests::context_after_decoding_text::context_after_decoding_text; - const LFM2_COLBERT_OUTPUT_PROJECTION_WIDTH: usize = 128; const QWEN3_RERANKER_CLASSIFIER_LABELS: usize = 2; diff --git a/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs b/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs index 24509fc55..d9d91c4c0 100644 --- a/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs +++ b/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::error::batch_transfer_error::BatchTransferError; use llama_cpp_bindings::error::encode_error::EncodeError; diff --git a/llama-cpp-bindings-tests/tests/encoder_requirement.rs b/llama-cpp-bindings-tests/tests/encoder_requirement.rs index 25d285cae..4d657a6e8 100644 --- a/llama-cpp-bindings-tests/tests/encoder_requirement.rs +++ b/llama-cpp-bindings-tests/tests/encoder_requirement.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::error::encode_error::EncodeError; diff --git a/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs b/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs index 17d5dbf63..bb183fd55 100644 --- a/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs +++ b/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs @@ -1,5 +1,6 @@ use anyhow::Context; use anyhow::Result; + use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/generation_end_classification.rs b/llama-cpp-bindings-tests/tests/generation_end_classification.rs index a6b4a5667..bfd336164 100644 --- a/llama-cpp-bindings-tests/tests/generation_end_classification.rs +++ b/llama-cpp-bindings-tests/tests/generation_end_classification.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::ingest_outcome::IngestOutcome; use llama_cpp_bindings::sampled_token::SampledToken; diff --git a/llama-cpp-bindings-tests/tests/grammar_sampler.rs b/llama-cpp-bindings-tests/tests/grammar_sampler.rs index d223c55b1..f9ad0db56 100644 --- a/llama-cpp-bindings-tests/tests/grammar_sampler.rs +++ b/llama-cpp-bindings-tests/tests/grammar_sampler.rs @@ -3,6 +3,7 @@ use std::ffi::CString; use anyhow::Context as _; use anyhow::Result; use anyhow::bail; + use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; diff --git a/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs b/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs index e1abff933..cb3e9e517 100644 --- a/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs +++ b/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; diff --git a/llama-cpp-bindings-tests/tests/image_question_answering.rs b/llama-cpp-bindings-tests/tests/image_question_answering.rs index 3df35469a..a0af4001d 100644 --- a/llama-cpp-bindings-tests/tests/image_question_answering.rs +++ b/llama-cpp-bindings-tests/tests/image_question_answering.rs @@ -1,5 +1,6 @@ use anyhow::Context; use anyhow::Result; + use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; diff --git a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs index 95818a80b..abca8fe3e 100644 --- a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs +++ b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs @@ -1,5 +1,6 @@ use anyhow::Context as _; use anyhow::Result; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::json_schema_to_grammar::json_schema_to_grammar; diff --git a/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs b/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs index c0d1b9df6..6ef59dd7d 100644 --- a/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs +++ b/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs @@ -1,6 +1,7 @@ use std::ffi::CString; use anyhow::Result; + use llama_cpp_bindings::error::grammar_error::GrammarError; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; diff --git a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs index 7230be30a..1e71ffaf9 100644 --- a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs +++ b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs @@ -2,6 +2,7 @@ use std::ffi::CStr; use anyhow::Context as _; use anyhow::Result; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::error::grammar_error::GrammarError; diff --git a/llama-cpp-bindings-tests/tests/logits_availability.rs b/llama-cpp-bindings-tests/tests/logits_availability.rs index 2a1dfda79..a94122170 100644 --- a/llama-cpp-bindings-tests/tests/logits_availability.rs +++ b/llama-cpp-bindings-tests/tests/logits_availability.rs @@ -1,15 +1,15 @@ use anyhow::Result; + use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; use llama_cpp_bindings::error::logits_error::LogitsError; use llama_cpp_bindings::error::sample_error::SampleError; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings::token::data_array::LlamaTokenDataArray; +use llama_cpp_bindings_tests::context_after_decoding_text::context_after_decoding_text; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; -use llama_cpp_bindings_tests::context_after_decoding_text::context_after_decoding_text; - #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), n_gpu_layers = 999, diff --git a/llama-cpp-bindings-tests/tests/logits_retrieval.rs b/llama-cpp-bindings-tests/tests/logits_retrieval.rs index c1710360b..672b17cf4 100644 --- a/llama-cpp-bindings-tests/tests/logits_retrieval.rs +++ b/llama-cpp-bindings-tests/tests/logits_retrieval.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::error::logits_error::LogitsError; diff --git a/llama-cpp-bindings-tests/tests/lora_adapters.rs b/llama-cpp-bindings-tests/tests/lora_adapters.rs index ec3d6bfb9..34ba6a43f 100644 --- a/llama-cpp-bindings-tests/tests/lora_adapters.rs +++ b/llama-cpp-bindings-tests/tests/lora_adapters.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::error::llama_lora_adapters_error::LlamaLoraAdaptersError; use llama_cpp_bindings::model::lora_adapter_scale::LoraAdapterScale; diff --git a/llama-cpp-bindings-tests/tests/memory_availability.rs b/llama-cpp-bindings-tests/tests/memory_availability.rs index 6ca886b7b..975776e31 100644 --- a/llama-cpp-bindings-tests/tests/memory_availability.rs +++ b/llama-cpp-bindings-tests/tests/memory_availability.rs @@ -2,6 +2,7 @@ use std::num::NonZeroU8; use anyhow::Context as _; use anyhow::Result; + use llama_cpp_bindings::error::memory_seq_add_error::MemorySeqAddError; use llama_cpp_bindings::error::memory_seq_cp_error::MemorySeqCpError; use llama_cpp_bindings::error::memory_seq_div_error::MemorySeqDivError; diff --git a/llama-cpp-bindings-tests/tests/memory_occupancy.rs b/llama-cpp-bindings-tests/tests/memory_occupancy.rs index cb8964d95..02614cd56 100644 --- a/llama-cpp-bindings-tests/tests/memory_occupancy.rs +++ b/llama-cpp-bindings-tests/tests/memory_occupancy.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings_tests::prime_memory::prime_memory; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/memory_position_bounds.rs b/llama-cpp-bindings-tests/tests/memory_position_bounds.rs index 5cc2e5d69..d8c27f5cc 100644 --- a/llama-cpp-bindings-tests/tests/memory_position_bounds.rs +++ b/llama-cpp-bindings-tests/tests/memory_position_bounds.rs @@ -2,6 +2,7 @@ use std::num::NonZeroU8; use anyhow::Context as _; use anyhow::Result; + use llama_cpp_bindings::error::memory_seq_add_error::MemorySeqAddError; use llama_cpp_bindings::error::memory_seq_cp_error::MemorySeqCpError; use llama_cpp_bindings::error::memory_seq_div_error::MemorySeqDivError; diff --git a/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs b/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs index 9ce951ce3..49a274fa0 100644 --- a/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs +++ b/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs @@ -2,6 +2,7 @@ use std::num::NonZeroU8; use anyhow::Context as _; use anyhow::Result; + use llama_cpp_bindings::error::memory_seq_add_error::MemorySeqAddError; use llama_cpp_bindings::error::memory_seq_div_error::MemorySeqDivError; use llama_cpp_bindings_tests::prime_memory::prime_memory; diff --git a/llama-cpp-bindings-tests/tests/memory_priming.rs b/llama-cpp-bindings-tests/tests/memory_priming.rs index e62104f4f..7a37c5586 100644 --- a/llama-cpp-bindings-tests/tests/memory_priming.rs +++ b/llama-cpp-bindings-tests/tests/memory_priming.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::batch_add_error::BatchAddError; use llama_cpp_bindings::error::string_to_token_error::StringToTokenError; use llama_cpp_bindings_tests::prime_memory_with::prime_memory_with; diff --git a/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs b/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs index 4c2d5c245..7dc303da0 100644 --- a/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs +++ b/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::error::memory_seq_rm_error::MemorySeqRmError; use llama_cpp_bindings_tests::prime_memory::prime_memory; use llama_cpp_test_harness::LlamaFixture; diff --git a/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs b/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs index f69d2cc93..d548ae979 100644 --- a/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs +++ b/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs @@ -1,4 +1,6 @@ use anyhow::Result; + +use llama_cpp_bindings::error::memory_seq_pos_max_error::MemorySeqPosMaxError; use llama_cpp_bindings::error::memory_seq_rm_error::MemorySeqRmError; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -42,7 +44,7 @@ fn memory_seq_pos_max_rejects_out_of_range_sequence(fixture: &LlamaFixture<'_>) assert_eq!( result, - Err(llama_cpp_bindings::error::memory_seq_pos_max_error::MemorySeqPosMaxError::SequenceIdOutOfRange { seq_id: 999 }) + Err(MemorySeqPosMaxError::SequenceIdOutOfRange { seq_id: 999 }) ); Ok(()) diff --git a/llama-cpp-bindings-tests/tests/model_architecture.rs b/llama-cpp-bindings-tests/tests/model_architecture.rs index 190087660..90224e9ab 100644 --- a/llama-cpp-bindings-tests/tests/model_architecture.rs +++ b/llama-cpp-bindings-tests/tests/model_architecture.rs @@ -1,3 +1,4 @@ +use llama_cpp_bindings::model::rope_type::RopeType; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -100,7 +101,6 @@ fn is_hybrid_returns_true_for_qwen_default_models(fixture: &LlamaFixture<'_>) { n_ubatch = 64, )] fn rope_type_returns_a_known_variant_for_rope_carrying_default_models(fixture: &LlamaFixture<'_>) { - use llama_cpp_bindings::model::rope_type::RopeType; let rope = fixture.model.rope_type(); assert!( matches!( @@ -128,7 +128,5 @@ fn rope_type_returns_a_known_variant_for_rope_carrying_default_models(fixture: & n_ubatch = 64, )] fn rope_type_reports_interleaved_mrope_for_qwen_default_models(fixture: &LlamaFixture<'_>) { - use llama_cpp_bindings::model::rope_type::RopeType; - assert_eq!(fixture.model.rope_type(), Some(RopeType::IMRope)); } diff --git a/llama-cpp-bindings-tests/tests/model_derived_data_caching.rs b/llama-cpp-bindings-tests/tests/model_derived_data_caching.rs index 0dc760f79..381fe5737 100644 --- a/llama-cpp-bindings-tests/tests/model_derived_data_caching.rs +++ b/llama-cpp-bindings-tests/tests/model_derived_data_caching.rs @@ -2,6 +2,7 @@ use std::ptr; use std::sync::Arc; use anyhow::Result; + use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/model_hyperparameters.rs b/llama-cpp-bindings-tests/tests/model_hyperparameters.rs index fcf7f6e22..da2410bf4 100644 --- a/llama-cpp-bindings-tests/tests/model_hyperparameters.rs +++ b/llama-cpp-bindings-tests/tests/model_hyperparameters.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/model_loading_errors.rs b/llama-cpp-bindings-tests/tests/model_loading_errors.rs index 0f7706962..2a6c93b0b 100644 --- a/llama-cpp-bindings-tests/tests/model_loading_errors.rs +++ b/llama-cpp-bindings-tests/tests/model_loading_errors.rs @@ -1,8 +1,13 @@ +use std::env; use std::ffi::CString; +use std::ffi::OsStr; +use std::fs; +use std::os::unix::ffi::OsStrExt; use std::path::Path; use std::path::PathBuf; use anyhow::Result; + use llama_cpp_bindings::error::llama_lora_adapter_init_error::LlamaLoraAdapterInitError; use llama_cpp_bindings::error::llama_model_load_error::LlamaModelLoadError; use llama_cpp_bindings::model::LlamaModel; @@ -41,8 +46,8 @@ fn load_model_with_invalid_file_content_returns_unloadable_or_reported( fixture: &LlamaFixture<'_>, ) -> Result<()> { let model_params = LlamaModelParams::default(); - let dummy_path = std::env::temp_dir().join("llama_test_invalid_model.gguf"); - std::fs::write(&dummy_path, b"not a valid gguf model file")?; + let dummy_path = env::temp_dir().join("llama_test_invalid_model.gguf"); + fs::write(&dummy_path, b"not a valid gguf model file")?; let result = LlamaModel::load_from_file(fixture.backend, &dummy_path, &model_params); @@ -50,7 +55,7 @@ fn load_model_with_invalid_file_content_returns_unloadable_or_reported( result.unwrap_err(), LlamaModelLoadError::Unloadable | LlamaModelLoadError::Reported { .. }, )); - let _ = std::fs::remove_file(&dummy_path); + let _ = fs::remove_file(&dummy_path); Ok(()) } @@ -63,9 +68,6 @@ fn load_model_with_invalid_file_content_returns_unloadable_or_reported( n_ubatch = 64, )] fn load_model_with_non_utf8_path_returns_path_to_str_error(fixture: &LlamaFixture<'_>) { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - let model_params = LlamaModelParams::default(); let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.gguf")); @@ -106,13 +108,13 @@ fn lora_adapter_init_with_invalid_path_returns_error(fixture: &LlamaFixture<'_>) fn lora_adapter_init_with_invalid_gguf_returns_unloadable( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let dummy_path = std::env::temp_dir().join("llama_test_dummy_lora.gguf"); - std::fs::write(&dummy_path, b"not a valid gguf")?; + let dummy_path = env::temp_dir().join("llama_test_dummy_lora.gguf"); + fs::write(&dummy_path, b"not a valid gguf")?; let result = fixture.model.lora_adapter_init(&dummy_path); assert_eq!(result.unwrap_err(), LlamaLoraAdapterInitError::Unloadable); - let _ = std::fs::remove_file(&dummy_path); + let _ = fs::remove_file(&dummy_path); Ok(()) } @@ -125,9 +127,6 @@ fn lora_adapter_init_with_invalid_gguf_returns_unloadable( n_ubatch = 64, )] fn lora_adapter_init_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.gguf")); let result = fixture.model.lora_adapter_init(non_utf8_path); diff --git a/llama-cpp-bindings-tests/tests/model_metadata.rs b/llama-cpp-bindings-tests/tests/model_metadata.rs index be2c9884c..9d46a3d69 100644 --- a/llama-cpp-bindings-tests/tests/model_metadata.rs +++ b/llama-cpp-bindings-tests/tests/model_metadata.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/multimodal_audio.rs b/llama-cpp-bindings-tests/tests/multimodal_audio.rs index 2d648e575..2f100e1df 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_audio.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_audio.rs @@ -1,5 +1,6 @@ use anyhow::Context; use anyhow::Result; + use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; use llama_cpp_bindings::chat_session::ChatSession; diff --git a/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs b/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs index ffe36334f..aee82d7c4 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs @@ -1,4 +1,7 @@ +use std::fs; + use anyhow::Result; + use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; use llama_cpp_test_harness::LlamaFixture; @@ -29,7 +32,7 @@ fn from_buffer_creates_bitmap_from_image_bytes(fixture: &LlamaFixture<'_>) -> Re let fixtures = fixtures_dir(); let image_path = fixtures.join("llamas.jpg"); - let image_bytes = std::fs::read(&image_path)?; + let image_bytes = fs::read(&image_path)?; let bitmap = MtmdBitmap::from_buffer(mtmd_ctx, &image_bytes)?; assert!(bitmap.nx() > 0); diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs index bbb5ed2fd..e6daaf7d9 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs @@ -1,5 +1,6 @@ use anyhow::Context as _; use anyhow::Result; + use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; use llama_cpp_test_harness::LlamaFixture; diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs index a7270b48d..8d24c7b5f 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs @@ -1,5 +1,6 @@ use anyhow::Context as _; use anyhow::Result; + use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; use llama_cpp_bindings::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs index 2adb7c53b..2861bfbcd 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs @@ -1,6 +1,7 @@ use anyhow::Context; use anyhow::Result; use anyhow::bail; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::error::eval_multimodal_chunks_error::EvalMultimodalChunksError; use llama_cpp_bindings::error::sequence_id_out_of_range::SequenceIdOutOfRange; @@ -21,14 +22,13 @@ use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; use llama_cpp_bindings_tests::media_question_chat_session::media_question_chat_session; use llama_cpp_bindings_tests::prime_memory_with::prime_memory_with; use llama_cpp_bindings_tests::prompt_question::PROMPT_QUESTION; +use llama_cpp_bindings_tests::tokenize_prompt_with_synthetic_image::tokenize_prompt_with_synthetic_image; use llama_cpp_fault_injection::ArmedFault; use llama_cpp_fault_injection::injected_fault_message; use llama_cpp_ffi_status::FfiContractError; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; -use llama_cpp_bindings_tests::tokenize_prompt_with_synthetic_image::tokenize_prompt_with_synthetic_image; - fn eval_synthetic_bitmap(fixture: &LlamaFixture<'_>, width: u32, height: u32) -> Result<()> { let mtmd_ctx = fixture .mtmd_context diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs index 817c51c22..707b6c91f 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs @@ -1,18 +1,18 @@ use anyhow::Context as _; use anyhow::Result; + use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; use llama_cpp_bindings::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; use llama_cpp_bindings::mtmd::mtmd_input_chunks_error::MtmdInputChunksError; use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; use llama_cpp_bindings::mtmd::mtmd_tokenize_error::MtmdTokenizeError; use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_bindings_tests::tokenize_prompt_with_synthetic_image::tokenize_prompt_with_synthetic_image; use llama_cpp_fault_injection::ArmedFault; use llama_cpp_fault_injection::injected_fault_message; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; -use llama_cpp_bindings_tests::tokenize_prompt_with_synthetic_image::tokenize_prompt_with_synthetic_image; - #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, diff --git a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs index 0f5866e0f..19a8d1975 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs @@ -1,5 +1,6 @@ use anyhow::Context; use anyhow::Result; + use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; use llama_cpp_bindings::chat_session::ChatSession; diff --git a/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs b/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs index a9b27c69c..6794d7e15 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs @@ -1,6 +1,7 @@ use anyhow::Context as _; use anyhow::Result; use anyhow::bail; + use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; use llama_cpp_bindings::mtmd::mtmd_capability::MtmdCapability; use llama_cpp_bindings::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; diff --git a/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs b/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs index f96b325d0..136ce05c1 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs @@ -2,6 +2,7 @@ use std::ffi::CString; use anyhow::Context as _; use anyhow::Result; + use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; diff --git a/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs index 0882725b5..775f34a92 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs @@ -1,6 +1,7 @@ use anyhow::Context as _; use anyhow::Result; use anyhow::bail; + use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; use llama_cpp_bindings::chat_session::ChatSession; diff --git a/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs b/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs index 03a7b4060..6a2b0237e 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs @@ -1,6 +1,7 @@ use anyhow::Context as _; use anyhow::Result; use anyhow::bail; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::eval_multimodal_chunks_params::EvalMultimodalChunksParams; use llama_cpp_bindings::mtmd::fitted_mtmd_input_chunk::FittedMtmdInputChunk; diff --git a/llama-cpp-bindings-tests/tests/parameter_fitting.rs b/llama-cpp-bindings-tests/tests/parameter_fitting.rs index e92070ec6..bbd54da42 100644 --- a/llama-cpp-bindings-tests/tests/parameter_fitting.rs +++ b/llama-cpp-bindings-tests/tests/parameter_fitting.rs @@ -3,6 +3,7 @@ use std::num::NonZeroU32; use std::pin::pin; use anyhow::Result; + use llama_cpp_bindings::context::params::LlamaContextParams; use llama_cpp_bindings::max_devices::max_devices; use llama_cpp_bindings::model::params::LlamaModelParams; diff --git a/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs b/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs index a25d238af..10a742e0e 100644 --- a/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs +++ b/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs @@ -1,5 +1,6 @@ use anyhow::Context as _; use anyhow::Result; + use llama_cpp_bindings::batch_add_error::BatchAddError; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; diff --git a/llama-cpp-bindings-tests/tests/reasoning_classification.rs b/llama-cpp-bindings-tests/tests/reasoning_classification.rs index 52370bf9d..af9ba1eea 100644 --- a/llama-cpp-bindings-tests/tests/reasoning_classification.rs +++ b/llama-cpp-bindings-tests/tests/reasoning_classification.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings::chat_session_options::ChatSessionOptions; diff --git a/llama-cpp-bindings-tests/tests/reranking.rs b/llama-cpp-bindings-tests/tests/reranking.rs index 7f15fa8e7..9b6226ec2 100644 --- a/llama-cpp-bindings-tests/tests/reranking.rs +++ b/llama-cpp-bindings-tests/tests/reranking.rs @@ -1,6 +1,7 @@ use anyhow::Context; use anyhow::Result; use anyhow::bail; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; @@ -61,7 +62,7 @@ fn reranking_produces_scores(fixture: &LlamaFixture<'_>) -> Result<()> { let tokens_lines_list = prompt_lines .iter() .map(|line| model.str_to_token(line, AddBos::Always, ParseSpecialTokens::Always)) - .collect::, _>>() + .collect::, _>>() .with_context(|| "failed to tokenize prompts")?; let n_ctx = usize::try_from(ctx.n_ctx())?; diff --git a/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs b/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs index f2ab2e65e..8b385a86c 100644 --- a/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs +++ b/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs @@ -1,6 +1,7 @@ use anyhow::Context as _; use anyhow::Result; use anyhow::bail; + use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::error::ingest_error::IngestError; diff --git a/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs b/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs index 20d4c2c0f..2c5d50c11 100644 --- a/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs +++ b/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs @@ -1,5 +1,6 @@ use anyhow::Context as _; use anyhow::Result; + use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings::token::LlamaToken; use llama_cpp_test_harness::LlamaFixture; diff --git a/llama-cpp-bindings-tests/tests/session_file_path_validation.rs b/llama-cpp-bindings-tests/tests/session_file_path_validation.rs index ccc5a5a34..9d0c4aeac 100644 --- a/llama-cpp-bindings-tests/tests/session_file_path_validation.rs +++ b/llama-cpp-bindings-tests/tests/session_file_path_validation.rs @@ -1,4 +1,9 @@ +use std::ffi::OsStr; +use std::os::unix::ffi::OsStrExt; +use std::path::Path; + use anyhow::Result; + use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -35,12 +40,9 @@ use llama_cpp_test_harness::llama_test; n_ubatch = 128, )] fn state_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - let context = fixture.build_context()?; - let non_utf8_path = std::path::Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); + let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); let result = context.state_save_file(non_utf8_path, &[]); assert!(result.is_err()); @@ -81,12 +83,9 @@ fn state_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) n_ubatch = 128, )] fn state_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - let mut context = fixture.build_context()?; - let non_utf8_path = std::path::Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); + let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); let result = context.state_load_file(non_utf8_path, 512); assert!(result.is_err()); @@ -127,12 +126,9 @@ fn state_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) n_ubatch = 128, )] fn state_seq_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - let context = fixture.build_context()?; - let non_utf8_path = std::path::Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); + let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); let result = context.state_seq_save_file(non_utf8_path, 0, &[]); assert!(result.is_err()); @@ -173,12 +169,9 @@ fn state_seq_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<' n_ubatch = 128, )] fn state_seq_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - let mut context = fixture.build_context()?; - let non_utf8_path = std::path::Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); + let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); let result = context.state_seq_load_file(non_utf8_path, 0, 512); assert!(result.is_err()); @@ -221,7 +214,7 @@ fn state_seq_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<' fn state_save_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { let context = fixture.build_context()?; - let path_with_null = std::path::Path::new("/tmp/foo\0bar.bin"); + let path_with_null = Path::new("/tmp/foo\0bar.bin"); let result = context.state_save_file(path_with_null, &[]); assert!(result.is_err()); @@ -264,7 +257,7 @@ fn state_save_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<' fn state_load_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; - let path_with_null = std::path::Path::new("/tmp/foo\0bar.bin"); + let path_with_null = Path::new("/tmp/foo\0bar.bin"); let result = context.state_load_file(path_with_null, 512); assert!(result.is_err()); @@ -309,7 +302,7 @@ fn state_seq_save_file_with_null_byte_in_path_returns_error( ) -> Result<()> { let context = fixture.build_context()?; - let path_with_null = std::path::Path::new("/tmp/foo\0bar.bin"); + let path_with_null = Path::new("/tmp/foo\0bar.bin"); let result = context.state_seq_save_file(path_with_null, 0, &[]); assert!(result.is_err()); @@ -354,7 +347,7 @@ fn state_seq_load_file_with_null_byte_in_path_returns_error( ) -> Result<()> { let mut context = fixture.build_context()?; - let path_with_null = std::path::Path::new("/tmp/foo\0bar.bin"); + let path_with_null = Path::new("/tmp/foo\0bar.bin"); let result = context.state_seq_load_file(path_with_null, 0, 512); assert!(result.is_err()); diff --git a/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs b/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs index ba646de5c..4f050b0f9 100644 --- a/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs +++ b/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs @@ -1,4 +1,8 @@ +use std::env; +use std::fs; + use anyhow::Result; + use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::add_bos::AddBos; @@ -49,13 +53,13 @@ fn state_load_file_with_zero_max_tokens_returns_error(fixture: &LlamaFixture<'_> batch.add_sequence(&tokens, 0, false)?; context.decode(&batch)?; - let session_path = std::env::temp_dir().join("llama_test_session_zero_max.bin"); + let session_path = env::temp_dir().join("llama_test_session_zero_max.bin"); context.state_save_file(&session_path, &tokens)?; let result = context.state_load_file(&session_path, 0); assert!(result.is_err()); - let _ = std::fs::remove_file(&session_path); + let _ = fs::remove_file(&session_path); Ok(()) } @@ -105,13 +109,13 @@ fn state_seq_load_file_with_zero_max_tokens_returns_error( batch.add_sequence(&tokens, 0, false)?; context.decode(&batch)?; - let session_path = std::env::temp_dir().join("llama_test_seq_state_zero_max.bin"); + let session_path = env::temp_dir().join("llama_test_seq_state_zero_max.bin"); context.state_seq_save_file(&session_path, 0, &tokens)?; let result = context.state_seq_load_file(&session_path, 0, 0); assert!(result.is_err()); - let _ = std::fs::remove_file(&session_path); + let _ = fs::remove_file(&session_path); Ok(()) } @@ -162,13 +166,13 @@ fn state_load_file_with_insufficient_max_tokens_returns_length_error( batch.add_sequence(&tokens, 0, false)?; context.decode(&batch)?; - let session_path = std::env::temp_dir().join("llama_test_session_insuf.bin"); + let session_path = env::temp_dir().join("llama_test_session_insuf.bin"); context.state_save_file(&session_path, &tokens)?; let result = context.state_load_file(&session_path, 1); assert!(result.is_err()); - let _ = std::fs::remove_file(&session_path); + let _ = fs::remove_file(&session_path); Ok(()) } @@ -219,13 +223,13 @@ fn state_seq_load_file_with_insufficient_max_tokens_returns_length_error( batch.add_sequence(&tokens, 0, false)?; context.decode(&batch)?; - let session_path = std::env::temp_dir().join("llama_test_seq_state_insuf.bin"); + let session_path = env::temp_dir().join("llama_test_seq_state_insuf.bin"); context.state_seq_save_file(&session_path, 0, &tokens)?; let result = context.state_seq_load_file(&session_path, 0, 1); assert!(result.is_err()); - let _ = std::fs::remove_file(&session_path); + let _ = fs::remove_file(&session_path); Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/session_files.rs b/llama-cpp-bindings-tests/tests/session_files.rs index 4f5930541..054138afd 100644 --- a/llama-cpp-bindings-tests/tests/session_files.rs +++ b/llama-cpp-bindings-tests/tests/session_files.rs @@ -1,4 +1,8 @@ +use std::env; +use std::fs; + use anyhow::Result; + use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::add_bos::AddBos; @@ -49,13 +53,13 @@ fn state_save_and_load_file_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> batch.add_sequence(&tokens, 0, false)?; context.decode(&batch)?; - let session_path = std::env::temp_dir().join("llama_test_session.bin"); + let session_path = env::temp_dir().join("llama_test_session.bin"); context.state_save_file(&session_path, &tokens)?; let loaded_tokens = context.state_load_file(&session_path, 512)?; assert_eq!(loaded_tokens, tokens); - std::fs::remove_file(&session_path)?; + fs::remove_file(&session_path)?; Ok(()) } @@ -103,7 +107,7 @@ fn state_seq_save_and_load_file_roundtrip(fixture: &LlamaFixture<'_>) -> Result< batch.add_sequence(&tokens, 0, false)?; context.decode(&batch)?; - let session_path = std::env::temp_dir().join("llama_test_seq_state.bin"); + let session_path = env::temp_dir().join("llama_test_seq_state.bin"); let bytes_written = context.state_seq_save_file(&session_path, 0, &tokens)?; assert!(bytes_written > 0); @@ -115,7 +119,7 @@ fn state_seq_save_and_load_file_roundtrip(fixture: &LlamaFixture<'_>) -> Result< "loading must consume exactly the bytes the save produced" ); - std::fs::remove_file(&session_path)?; + fs::remove_file(&session_path)?; Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/special_tokens.rs b/llama-cpp-bindings-tests/tests/special_tokens.rs index d6e330ed3..f24a7c64a 100644 --- a/llama-cpp-bindings-tests/tests/special_tokens.rs +++ b/llama-cpp-bindings-tests/tests/special_tokens.rs @@ -1,5 +1,6 @@ use anyhow::Context; use anyhow::Result; + use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampled_token::SampledToken; diff --git a/llama-cpp-bindings-tests/tests/state_snapshots.rs b/llama-cpp-bindings-tests/tests/state_snapshots.rs index 079b4bf35..e5179e658 100644 --- a/llama-cpp-bindings-tests/tests/state_snapshots.rs +++ b/llama-cpp-bindings-tests/tests/state_snapshots.rs @@ -1,5 +1,8 @@ use anyhow::Result; + use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::context::llama_state_seq_flags::LlamaStateSeqFlags; +use llama_cpp_bindings::context::state_data_error::StateDataError; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; @@ -95,11 +98,9 @@ fn state_set_data_rejects_a_truncated_snapshot(fixture: &LlamaFixture<'_>) -> Re assert_eq!( unsafe { context.state_set_data(&state_data) }, - Err( - llama_cpp_bindings::context::state_data_error::StateDataError::NothingRestored { - provided_bytes: state_data.len(), - } - ), + Err(StateDataError::NothingRestored { + provided_bytes: state_data.len(), + }), "a truncated snapshot restores nothing, and that must not be reported as success" ); @@ -174,8 +175,6 @@ fn state_get_data_and_state_set_data_roundtrip(fixture: &LlamaFixture<'_>) -> Re fn state_seq_get_size_ext_returns_size_for_decoded_sequence( fixture: &LlamaFixture<'_>, ) -> Result<()> { - use llama_cpp_bindings::context::llama_state_seq_flags::LlamaStateSeqFlags; - let mut context = fixture.build_context()?; let tokens = @@ -227,8 +226,6 @@ fn state_seq_get_size_ext_returns_size_for_decoded_sequence( n_ubatch = 128, )] fn state_seq_get_data_ext_and_set_data_ext_round_trip(fixture: &LlamaFixture<'_>) -> Result<()> { - use llama_cpp_bindings::context::llama_state_seq_flags::LlamaStateSeqFlags; - let mut context = fixture.build_context()?; let tokens = diff --git a/llama-cpp-bindings-tests/tests/threadpool_detachment.rs b/llama-cpp-bindings-tests/tests/threadpool_detachment.rs index d4f2cdd66..879dc7aa9 100644 --- a/llama-cpp-bindings-tests/tests/threadpool_detachment.rs +++ b/llama-cpp-bindings-tests/tests/threadpool_detachment.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/token_attributes.rs b/llama-cpp-bindings-tests/tests/token_attributes.rs index dac76b5d3..629303028 100644 --- a/llama-cpp-bindings-tests/tests/token_attributes.rs +++ b/llama-cpp-bindings-tests/tests/token_attributes.rs @@ -1,5 +1,6 @@ use anyhow::Context; use anyhow::Result; + use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/token_candidates.rs b/llama-cpp-bindings-tests/tests/token_candidates.rs index 1c84cc0d3..854a6afce 100644 --- a/llama-cpp-bindings-tests/tests/token_candidates.rs +++ b/llama-cpp-bindings-tests/tests/token_candidates.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; diff --git a/llama-cpp-bindings-tests/tests/tokenization.rs b/llama-cpp-bindings-tests/tests/tokenization.rs index 5bf4f5d0e..c0751d4e6 100644 --- a/llama-cpp-bindings-tests/tests/tokenization.rs +++ b/llama-cpp-bindings-tests/tests/tokenization.rs @@ -1,4 +1,7 @@ +use std::fmt::Write; + use anyhow::Result; + use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampled_token::SampledToken; @@ -180,8 +183,6 @@ fn str_to_token_without_bos_yields_no_more_tokens_than_with_bos( n_ubatch = 64, )] fn str_to_token_with_many_tokens_triggers_buffer_resize(fixture: &LlamaFixture<'_>) -> Result<()> { - use std::fmt::Write; - let many_numbers = (0..2000).fold(String::new(), |mut accumulator, number| { let _ = write!(accumulator, "{number} "); accumulator diff --git a/llama-cpp-bindings-tests/tests/tool_call_classification.rs b/llama-cpp-bindings-tests/tests/tool_call_classification.rs index 1394075fb..2576e1874 100644 --- a/llama-cpp-bindings-tests/tests/tool_call_classification.rs +++ b/llama-cpp-bindings-tests/tests/tool_call_classification.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; use llama_cpp_bindings::ingest_outcome::IngestOutcome; diff --git a/llama-cpp-bindings-tests/tests/tool_call_grammar.rs b/llama-cpp-bindings-tests/tests/tool_call_grammar.rs index 29ba0d490..87aa24d01 100644 --- a/llama-cpp-bindings-tests/tests/tool_call_grammar.rs +++ b/llama-cpp-bindings-tests/tests/tool_call_grammar.rs @@ -1,5 +1,6 @@ use anyhow::Context as _; use anyhow::Result; + use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; diff --git a/llama-cpp-bindings-tests/tests/tool_call_parsing.rs b/llama-cpp-bindings-tests/tests/tool_call_parsing.rs index 0feaaba93..9d8982c1f 100644 --- a/llama-cpp-bindings-tests/tests/tool_call_parsing.rs +++ b/llama-cpp-bindings-tests/tests/tool_call_parsing.rs @@ -1,4 +1,6 @@ use anyhow::Result; +use serde_json::json; + use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; use llama_cpp_bindings::chat_session::ChatSession; @@ -14,7 +16,6 @@ use llama_cpp_bindings_types::ParsedToolCallId; use llama_cpp_bindings_types::ToolCallArguments; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; -use serde_json::json; const WEATHER_QUESTION: &str = "What is the weather in Paris?"; const PLAIN_CONTENT_REFUSAL: &str = "Sorry, I cannot help with that."; diff --git a/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs b/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs index 424bf5e15..d81bbdde4 100644 --- a/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs +++ b/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::generation_progress::GenerationProgress; diff --git a/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs b/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs index 3ae9c1a8b..156e2e67f 100644 --- a/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs +++ b/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; diff --git a/llama-cpp-bindings-tests/tests/vocabulary_type.rs b/llama-cpp-bindings-tests/tests/vocabulary_type.rs index 44c0e1c76..23ade80fb 100644 --- a/llama-cpp-bindings-tests/tests/vocabulary_type.rs +++ b/llama-cpp-bindings-tests/tests/vocabulary_type.rs @@ -1,4 +1,6 @@ use anyhow::Result; + +use llama_cpp_bindings::model::vocab_type::VocabType; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -35,8 +37,6 @@ use llama_cpp_test_harness::llama_test; n_ubatch = 64, )] fn default_models_use_a_byte_pair_encoding_vocabulary(fixture: &LlamaFixture<'_>) -> Result<()> { - use llama_cpp_bindings::model::vocab_type::VocabType; - assert_eq!(fixture.model.vocab_type()?, VocabType::Bpe); Ok(()) diff --git a/llama-cpp-bindings-types/src/token_usage.rs b/llama-cpp-bindings-types/src/token_usage.rs index 6cc1ac6ad..55f9a90e1 100644 --- a/llama-cpp-bindings-types/src/token_usage.rs +++ b/llama-cpp-bindings-types/src/token_usage.rs @@ -154,6 +154,8 @@ impl<'usage> Sum<&'usage Self> for TokenUsage { #[cfg(test)] mod tests { + use std::iter; + use super::TokenUsage; use super::TokenUsageError; @@ -442,7 +444,7 @@ mod tests { #[test] fn sum_over_empty_iter_returns_default() { - let summed: TokenUsage = std::iter::empty::().sum(); + let summed: TokenUsage = iter::empty::().sum(); assert_eq!(summed, TokenUsage::default()); } diff --git a/llama-cpp-bindings/build.rs b/llama-cpp-bindings/build.rs index fa6334b55..5294abc51 100644 --- a/llama-cpp-bindings/build.rs +++ b/llama-cpp-bindings/build.rs @@ -1,5 +1,7 @@ +use std::env; + fn main() { - if let Ok(dir) = std::env::var("DEP_LLAMA_BACKENDS_DIR") { + if let Ok(dir) = env::var("DEP_LLAMA_BACKENDS_DIR") { println!("cargo:rustc-env=GGML_BACKENDS_DIR={dir}"); } } diff --git a/llama-cpp-bindings/src/chat_message_parse_outcome.rs b/llama-cpp-bindings/src/chat_message_parse_outcome.rs index e4000db4d..64762a72f 100644 --- a/llama-cpp-bindings/src/chat_message_parse_outcome.rs +++ b/llama-cpp-bindings/src/chat_message_parse_outcome.rs @@ -37,11 +37,12 @@ impl ChatMessageParseOutcome { #[cfg(test)] mod tests { + use serde_json::json; + use llama_cpp_bindings_types::ParsedChatMessage; use llama_cpp_bindings_types::ParsedToolCall; use llama_cpp_bindings_types::ParsedToolCallId; use llama_cpp_bindings_types::ToolCallArguments; - use serde_json::json; use crate::chat_message_parse_outcome::ChatMessageParseOutcome; use crate::chat_output_completeness::ChatOutputCompleteness; diff --git a/llama-cpp-bindings/src/chat_session.rs b/llama-cpp-bindings/src/chat_session.rs index a13154cb6..7d746720f 100644 --- a/llama-cpp-bindings/src/chat_session.rs +++ b/llama-cpp-bindings/src/chat_session.rs @@ -4,11 +4,12 @@ use std::ptr; use std::ptr::NonNull; use std::slice; -use llama_cpp_bindings_types::ParsedChatMessage; -use llama_cpp_ffi_status::read_and_free_cpp_string; use serde_json::Value; use serde_json::json; +use llama_cpp_bindings_types::ParsedChatMessage; +use llama_cpp_ffi_status::read_and_free_cpp_string; + use crate::chat_assistant_prefill::ChatAssistantPrefill; use crate::chat_assistant_turn::ChatAssistantTurn; use crate::chat_classification::ChatClassification; diff --git a/llama-cpp-bindings/src/chat_spans.rs b/llama-cpp-bindings/src/chat_spans.rs index cb8031b0c..22aaa04e9 100644 --- a/llama-cpp-bindings/src/chat_spans.rs +++ b/llama-cpp-bindings/src/chat_spans.rs @@ -74,14 +74,13 @@ impl ChatSpans { #[cfg(test)] mod tests { + use super::ChatSpans; use crate::chat_output_completeness::ChatOutputCompleteness; use crate::chat_piece_role::ChatPieceRole; use crate::chat_span::ChatSpan; use crate::chat_span_kind::ChatSpanKind; use crate::sampled_token_section::SampledTokenSection; - use super::ChatSpans; - fn span(kind: ChatSpanKind, start: usize, end: usize) -> ChatSpan { ChatSpan { kind, diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index 27ae33f20..698cc0f10 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -1,5 +1,25 @@ +pub mod context_limits; +pub mod decoded_output_rows; +pub mod decoded_outputs; +pub mod kv_cache_type; +pub mod llama_attention_type; +pub mod llama_pooling_type; +pub mod llama_state_seq_flags; +pub mod load_seq_state_error; +pub mod load_session_error; +pub mod loaded_seq_state; +pub mod memory; +pub mod params; +pub mod rope_scaling_type; +pub mod save_seq_state_error; +pub mod save_session_error; +mod sequence_output_capacity; +pub mod session; +pub mod state_data_error; + use std::ffi::c_char; use std::ffi::c_void; +use std::fmt; use std::fmt::{Debug, Formatter}; use std::num::NonZeroI32; use std::ptr; @@ -372,25 +392,6 @@ unsafe fn logits_slice<'logits>( Ok(unsafe { slice::from_raw_parts(logits.as_ptr(), len) }) } -pub mod context_limits; -pub mod decoded_output_rows; -pub mod decoded_outputs; -pub mod kv_cache_type; -pub mod llama_attention_type; -pub mod llama_pooling_type; -pub mod llama_state_seq_flags; -pub mod load_seq_state_error; -pub mod load_session_error; -pub mod loaded_seq_state; -pub mod memory; -pub mod params; -pub mod rope_scaling_type; -pub mod save_seq_state_error; -pub mod save_session_error; -mod sequence_output_capacity; -pub mod session; -pub mod state_data_error; - unsafe extern "C" fn abort_callback_trampoline(data: *mut c_void) -> bool { let flag = unsafe { &*(data as *const AtomicBool) }; @@ -408,7 +409,7 @@ pub struct LlamaContext<'model> { } impl Debug for LlamaContext<'_> { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { f.debug_struct("LlamaContext") .field("context", &self.context) .finish() diff --git a/llama-cpp-bindings/src/context/llama_state_seq_flags.rs b/llama-cpp-bindings/src/context/llama_state_seq_flags.rs index cbe5ccc09..98c3a26f2 100644 --- a/llama-cpp-bindings/src/context/llama_state_seq_flags.rs +++ b/llama-cpp-bindings/src/context/llama_state_seq_flags.rs @@ -1,3 +1,5 @@ +use std::ops::BitOr; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct LlamaStateSeqFlags { flags: u32, @@ -22,7 +24,7 @@ impl LlamaStateSeqFlags { } } -impl std::ops::BitOr for LlamaStateSeqFlags { +impl BitOr for LlamaStateSeqFlags { type Output = Self; fn bitor(self, rhs: Self) -> Self { diff --git a/llama-cpp-bindings/src/context/memory.rs b/llama-cpp-bindings/src/context/memory.rs index 4811a085d..552d92a4e 100644 --- a/llama-cpp-bindings/src/context/memory.rs +++ b/llama-cpp-bindings/src/context/memory.rs @@ -3,13 +3,14 @@ use std::num::NonZeroU8; use std::os::raw::c_char; use std::ptr; +use llama_cpp_ffi_status::read_and_free_cpp_string; + use crate::context::LlamaContext; use crate::error::memory_seq_add_error::MemorySeqAddError; use crate::error::memory_seq_cp_error::MemorySeqCpError; use crate::error::memory_seq_div_error::MemorySeqDivError; use crate::error::memory_seq_pos_max_error::MemorySeqPosMaxError; use crate::error::memory_seq_rm_error::MemorySeqRmError; -use llama_cpp_ffi_status::read_and_free_cpp_string; fn memory_seq_add_status_to_result( status: llama_cpp_bindings_sys::llama_rs_memory_seq_add_status, diff --git a/llama-cpp-bindings/src/context/params.rs b/llama-cpp-bindings/src/context/params.rs index d2636fafe..264ca3162 100644 --- a/llama-cpp-bindings/src/context/params.rs +++ b/llama-cpp-bindings/src/context/params.rs @@ -1,9 +1,11 @@ +use std::ffi::c_void; +use std::fmt::Debug; +use std::num::NonZeroU32; + use crate::context::kv_cache_type::KvCacheType; use crate::context::llama_attention_type::LlamaAttentionType; use crate::context::llama_pooling_type::LlamaPoolingType; use crate::context::rope_scaling_type::RopeScalingType; -use std::fmt::Debug; -use std::num::NonZeroU32; #[derive(Debug, Clone, Copy)] pub struct LlamaContextParams { @@ -148,10 +150,7 @@ impl LlamaContextParams { } #[must_use] - pub const fn with_cb_eval_user_data( - mut self, - cb_eval_user_data: *mut std::ffi::c_void, - ) -> Self { + pub const fn with_cb_eval_user_data(mut self, cb_eval_user_data: *mut c_void) -> Self { self.context_params.cb_eval_user_data = cb_eval_user_data; self } @@ -332,6 +331,10 @@ impl Default for LlamaContextParams { #[cfg(test)] mod tests { + use std::ffi::c_void; + use std::num::NonZeroU32; + use std::ptr; + use super::KvCacheType; use super::LlamaAttentionType; use super::LlamaPoolingType; @@ -341,7 +344,7 @@ mod tests { fn default_params_have_expected_values() { let params = super::LlamaContextParams::default(); - assert_eq!(params.n_ctx(), std::num::NonZeroU32::new(512)); + assert_eq!(params.n_ctx(), NonZeroU32::new(512)); assert_eq!(params.n_batch(), 2048); assert_eq!(params.n_ubatch(), 512); assert_eq!(params.rope_scaling_type(), RopeScalingType::Unspecified); @@ -350,10 +353,9 @@ mod tests { #[test] fn with_n_ctx_sets_value() { - let params = - super::LlamaContextParams::default().with_n_ctx(std::num::NonZeroU32::new(2048)); + let params = super::LlamaContextParams::default().with_n_ctx(NonZeroU32::new(2048)); - assert_eq!(params.n_ctx(), std::num::NonZeroU32::new(2048)); + assert_eq!(params.n_ctx(), NonZeroU32::new(2048)); } #[test] @@ -548,7 +550,7 @@ mod tests { #[test] fn builder_chaining_preserves_all_values() { let params = super::LlamaContextParams::default() - .with_n_ctx(std::num::NonZeroU32::new(1024)) + .with_n_ctx(NonZeroU32::new(1024)) .with_n_batch(4096) .with_n_ubatch(256) .with_n_threads(8) @@ -559,7 +561,7 @@ mod tests { .with_rope_freq_base(5000.0) .with_rope_freq_scale(0.25); - assert_eq!(params.n_ctx(), std::num::NonZeroU32::new(1024)); + assert_eq!(params.n_ctx(), NonZeroU32::new(1024)); assert_eq!(params.n_batch(), 4096); assert_eq!(params.n_ubatch(), 256); assert_eq!(params.n_threads(), 8); @@ -576,12 +578,12 @@ mod tests { extern "C" fn test_cb_eval( _tensor: *mut llama_cpp_bindings_sys::ggml_tensor, _ask: bool, - _user_data: *mut std::ffi::c_void, + _user_data: *mut c_void, ) -> bool { false } - let result = test_cb_eval(std::ptr::null_mut(), false, std::ptr::null_mut()); + let result = test_cb_eval(ptr::null_mut(), false, ptr::null_mut()); assert!(!result); @@ -593,7 +595,7 @@ mod tests { #[test] fn with_cb_eval_user_data_sets_pointer() { let mut value: i32 = 42; - let user_data = (&raw mut value).cast::(); + let user_data = (&raw mut value).cast::(); let params = super::LlamaContextParams::default().with_cb_eval_user_data(user_data); assert_eq!(params.context_params.cb_eval_user_data, user_data); diff --git a/llama-cpp-bindings/src/gguf_context.rs b/llama-cpp-bindings/src/gguf_context.rs index a96601920..787c81291 100644 --- a/llama-cpp-bindings/src/gguf_context.rs +++ b/llama-cpp-bindings/src/gguf_context.rs @@ -4,6 +4,7 @@ use std::fs::File; use std::io; use std::os::unix::fs::FileExt as _; use std::path::{Path, PathBuf}; +use std::ptr; use std::ptr::NonNull; use std::slice; @@ -34,7 +35,7 @@ impl GgufContext { let init_params = llama_cpp_bindings_sys::gguf_init_params { no_alloc: true, - ctx: std::ptr::null_mut(), + ctx: ptr::null_mut(), }; let raw = @@ -400,9 +401,20 @@ impl Drop for GgufContext { #[cfg(test)] mod tests { + use std::env; use std::ffi::CString; + use std::ffi::OsStr; + use std::fs; + use std::fs::File; + use std::io; + use std::io::Write as _; use std::mem::Discriminant; + use std::mem::discriminant; + use std::os::unix::ffi::OsStrExt; + use std::path::Path; use std::path::PathBuf; + use std::process; + use std::str; use super::GgufContext; use crate::gguf_context_error::GgufContextError; @@ -418,28 +430,28 @@ mod tests { } fn init_failed_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::InitFailed(PathBuf::new())) + discriminant(&GgufContextError::InitFailed(PathBuf::new())) } fn key_not_found_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::KeyNotFound { key: String::new() }) + discriminant(&GgufContextError::KeyNotFound { key: String::new() }) } fn nul_error_disc() -> Discriminant { let nul_err = CString::new(b"a\0b".to_vec()).unwrap_err(); - std::mem::discriminant(&GgufContextError::NulError(nul_err)) + discriminant(&GgufContextError::NulError(nul_err)) } fn path_to_str_error_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::PathToStrError(PathBuf::new())) + discriminant(&GgufContextError::PathToStrError(PathBuf::new())) } fn key_id_out_of_range_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::KeyIdOutOfRange { key_id: 0, n_kv: 0 }) + discriminant(&GgufContextError::KeyIdOutOfRange { key_id: 0, n_kv: 0 }) } fn value_type_mismatch_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::ValueTypeMismatch { + discriminant(&GgufContextError::ValueTypeMismatch { key_id: 0, expected: GgufType::Uint32, actual: GgufType::Uint32, @@ -447,20 +459,20 @@ mod tests { } fn tensor_not_found_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::TensorNotFound { + discriminant(&GgufContextError::TensorNotFound { name: String::new(), }) } fn tensor_is_not_f32_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::TensorIsNotF32 { + discriminant(&GgufContextError::TensorIsNotF32 { name: String::new(), ggml_type: 0, }) } fn array_element_type_mismatch_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::ArrayElementTypeMismatch { + discriminant(&GgufContextError::ArrayElementTypeMismatch { key_id: 0, expected: GgufType::Int32, actual_raw_type: 0, @@ -480,14 +492,14 @@ mod tests { } fn source_file_read_failed_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::SourceFileReadFailed { + discriminant(&GgufContextError::SourceFileReadFailed { path: PathBuf::new(), - source: std::io::Error::other("discriminant"), + source: io::Error::other("discriminant"), }) } fn source_file_truncated_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::SourceFileTruncated { + discriminant(&GgufContextError::SourceFileTruncated { path: PathBuf::new(), data_offset: 0, file_size: 0, @@ -495,32 +507,32 @@ mod tests { } fn destination_file_write_failed_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::DestinationFileWriteFailed { + discriminant(&GgufContextError::DestinationFileWriteFailed { path: PathBuf::new(), - source: std::io::Error::other("discriminant"), + source: io::Error::other("discriminant"), }) } fn rewritten_gguf_path(test_name: &str) -> PathBuf { - std::env::temp_dir().join(format!( + env::temp_dir().join(format!( "llama_cpp_bindings_rewritten_{}_{}.gguf", - std::process::id(), + process::id(), test_name, )) } fn tensor_read_failed_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::TensorReadFailed { + discriminant(&GgufContextError::TensorReadFailed { name: String::new(), path: PathBuf::new(), - source: std::io::Error::other("discriminant"), + source: io::Error::other("discriminant"), }) } fn utf8_error_disc() -> Discriminant { let invalid_utf8_bytes: Vec = vec![0xFF]; - let utf8_err = std::str::from_utf8(&invalid_utf8_bytes).unwrap_err(); - std::mem::discriminant(&GgufContextError::Utf8Error(utf8_err)) + let utf8_err = str::from_utf8(&invalid_utf8_bytes).unwrap_err(); + discriminant(&GgufContextError::Utf8Error(utf8_err)) } struct SyntheticTensor { @@ -603,15 +615,13 @@ mod tests { impl SyntheticGgufFile { fn from_bytes(test_name: &str, bytes: &[u8]) -> Self { - use std::io::Write as _; - - let path = std::env::temp_dir().join(format!( + let path = env::temp_dir().join(format!( "llama_cpp_bindings_synthetic_{}_{}.gguf", - std::process::id(), + process::id(), test_name, )); - let mut file = std::fs::File::create(&path).unwrap(); + let mut file = File::create(&path).unwrap(); file.write_all(bytes).unwrap(); Self { path } @@ -620,7 +630,7 @@ mod tests { impl Drop for SyntheticGgufFile { fn drop(&mut self) { - std::fs::remove_file(&self.path).unwrap(); + fs::remove_file(&self.path).unwrap(); } } @@ -642,7 +652,7 @@ mod tests { fn from_file_nonexistent_returns_init_failed() { let err = GgufContext::from_file("/nonexistent/file.gguf").unwrap_err(); - assert_eq!(std::mem::discriminant(&err), init_failed_disc()); + assert_eq!(discriminant(&err), init_failed_disc()); } #[test] @@ -666,7 +676,7 @@ mod tests { let context = GgufContext::from_file(fixture_path()).unwrap(); let err = context.find_key("nonexistent.key").unwrap_err(); - assert_eq!(std::mem::discriminant(&err), key_not_found_disc()); + assert_eq!(discriminant(&err), key_not_found_disc()); } #[test] @@ -683,7 +693,7 @@ mod tests { let context = GgufContext::from_file(fixture_path()).unwrap(); let err = context.key_at(context.n_kv()).unwrap_err(); - assert_eq!(std::mem::discriminant(&err), key_id_out_of_range_disc()); + assert_eq!(discriminant(&err), key_id_out_of_range_disc()); } #[test] @@ -691,7 +701,7 @@ mod tests { let context = GgufContext::from_file(fixture_path()).unwrap(); let err = context.val_u32(-1).unwrap_err(); - assert_eq!(std::mem::discriminant(&err), key_id_out_of_range_disc()); + assert_eq!(discriminant(&err), key_id_out_of_range_disc()); } #[test] @@ -713,20 +723,17 @@ mod tests { #[test] fn from_file_non_utf8_path_returns_error() { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - - let non_utf8_path = std::path::Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.gguf")); + let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.gguf")); let err = GgufContext::from_file(non_utf8_path).unwrap_err(); - assert_eq!(std::mem::discriminant(&err), path_to_str_error_disc()); + assert_eq!(discriminant(&err), path_to_str_error_disc()); } #[test] fn from_file_with_null_byte_in_path_returns_error() { let err = GgufContext::from_file("/tmp/foo\0bar.gguf").unwrap_err(); - assert_eq!(std::mem::discriminant(&err), nul_error_disc()); + assert_eq!(discriminant(&err), nul_error_disc()); } #[test] @@ -734,7 +741,7 @@ mod tests { let context = GgufContext::from_file(fixture_path()).unwrap(); let err = context.find_key("foo\0bar").unwrap_err(); - assert_eq!(std::mem::discriminant(&err), nul_error_disc()); + assert_eq!(discriminant(&err), nul_error_disc()); } #[test] @@ -790,10 +797,7 @@ mod tests { ]; for mismatch in mismatches { - assert_eq!( - std::mem::discriminant(&mismatch), - value_type_mismatch_disc() - ); + assert_eq!(discriminant(&mismatch), value_type_mismatch_disc()); } } @@ -807,7 +811,7 @@ mod tests { let value_index = context.find_key("synthetic.str_value").unwrap(); let err = context.val_str(value_index).unwrap_err(); - assert_eq!(std::mem::discriminant(&err), utf8_error_disc()); + assert_eq!(discriminant(&err), utf8_error_disc()); } #[test] @@ -819,7 +823,7 @@ mod tests { let err = context.key_at(0).unwrap_err(); - assert_eq!(std::mem::discriminant(&err), utf8_error_disc()); + assert_eq!(discriminant(&err), utf8_error_disc()); } #[test] @@ -847,7 +851,7 @@ mod tests { let context = GgufContext::from_file(fixture_path()).unwrap(); let err = context.read_tensor_f32("synthetic.missing").unwrap_err(); - assert_eq!(std::mem::discriminant(&err), tensor_not_found_disc()); + assert_eq!(discriminant(&err), tensor_not_found_disc()); } #[test] @@ -858,7 +862,7 @@ mod tests { let context = GgufContext::from_file(&fixture.path).unwrap(); let err = context.read_tensor_f32("synthetic.half").unwrap_err(); - assert_eq!(std::mem::discriminant(&err), tensor_is_not_f32_disc()); + assert_eq!(discriminant(&err), tensor_is_not_f32_disc()); } #[test] @@ -873,11 +877,11 @@ mod tests { .write("read_tensor_f32_reports_a_file_truncated_after_opening"); let context = GgufContext::from_file(&fixture.path).unwrap(); - std::fs::write(&fixture.path, b"").unwrap(); + fs::write(&fixture.path, b"").unwrap(); let err = context.read_tensor_f32("synthetic.weight").unwrap_err(); - assert_eq!(std::mem::discriminant(&err), tensor_read_failed_disc()); + assert_eq!(discriminant(&err), tensor_read_failed_disc()); } #[test] @@ -892,14 +896,14 @@ mod tests { .write("read_tensor_f32_reports_a_file_removed_after_opening"); let context = GgufContext::from_file(&fixture.path).unwrap(); - std::fs::remove_file(&fixture.path).unwrap(); + fs::remove_file(&fixture.path).unwrap(); let result = context.read_tensor_f32("synthetic.weight"); - std::fs::write(&fixture.path, b"").unwrap(); + fs::write(&fixture.path, b"").unwrap(); assert_eq!( - std::mem::discriminant(&result.unwrap_err()), + discriminant(&result.unwrap_err()), tensor_read_failed_disc() ); } @@ -909,7 +913,7 @@ mod tests { let context = GgufContext::from_file(fixture_path()).unwrap(); let err = context.read_tensor_f32("foo\0bar").unwrap_err(); - assert_eq!(std::mem::discriminant(&err), nul_error_disc()); + assert_eq!(discriminant(&err), nul_error_disc()); } #[test] @@ -933,7 +937,7 @@ mod tests { let index = context.find_key("general.architecture").unwrap(); let err = context.val_arr_i32(index).unwrap_err(); - assert_eq!(std::mem::discriminant(&err), value_type_mismatch_disc()); + assert_eq!(discriminant(&err), value_type_mismatch_disc()); } #[test] @@ -950,10 +954,7 @@ mod tests { let array_index = context.find_key("synthetic.u8_array").unwrap(); let err = context.val_arr_i32(array_index).unwrap_err(); - assert_eq!( - std::mem::discriminant(&err), - array_element_type_mismatch_disc() - ); + assert_eq!(discriminant(&err), array_element_type_mismatch_disc()); } #[test] @@ -961,7 +962,7 @@ mod tests { let mut context = GgufContext::from_file(fixture_path()).unwrap(); let err = context.remove_key("nonexistent.key").unwrap_err(); - assert_eq!(std::mem::discriminant(&err), key_not_found_disc()); + assert_eq!(discriminant(&err), key_not_found_disc()); } #[test] @@ -988,11 +989,11 @@ mod tests { let removed_err = rewritten.find_key("synthetic.removed").unwrap_err(); let tensor = rewritten.read_tensor_f32("synthetic.weight").unwrap(); - std::fs::remove_file(&rewritten_path).unwrap(); + fs::remove_file(&rewritten_path).unwrap(); assert_eq!(rewritten.n_kv(), 1); assert_eq!(rewritten.val_str(kept_index).unwrap(), "kept value"); - assert_eq!(std::mem::discriminant(&removed_err), key_not_found_disc()); + assert_eq!(discriminant(&removed_err), key_not_found_disc()); assert_eq!(tensor.shape, [3, 2, 1, 1]); assert_eq!(tensor.values, vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]); } @@ -1020,7 +1021,7 @@ mod tests { let str_index = rewritten.find_key("synthetic.str_value").unwrap(); let array_index = rewritten.find_key("synthetic.i32_array").unwrap(); - std::fs::remove_file(&rewritten_path).unwrap(); + fs::remove_file(&rewritten_path).unwrap(); assert_eq!(rewritten.val_u32(u32_index).unwrap(), 4096); assert_eq!(rewritten.val_str(str_index).unwrap(), "set value"); @@ -1034,16 +1035,16 @@ mod tests { .write("write_to_file_reports_a_source_file_removed_after_opening"); let context = GgufContext::from_file(&fixture.path).unwrap(); - std::fs::remove_file(&fixture.path).unwrap(); + fs::remove_file(&fixture.path).unwrap(); let result = context.write_to_file(rewritten_gguf_path( "write_to_file_reports_a_source_file_removed_after_opening", )); - std::fs::write(&fixture.path, b"").unwrap(); + fs::write(&fixture.path, b"").unwrap(); assert_eq!( - std::mem::discriminant(&result.unwrap_err()), + discriminant(&result.unwrap_err()), source_file_read_failed_disc() ); } @@ -1060,7 +1061,7 @@ mod tests { .write("write_to_file_reports_a_source_file_truncated_before_its_tensor_data"); let context = GgufContext::from_file(&fixture.path).unwrap(); - std::fs::write(&fixture.path, b"").unwrap(); + fs::write(&fixture.path, b"").unwrap(); let err = context .write_to_file(rewritten_gguf_path( @@ -1068,7 +1069,7 @@ mod tests { )) .unwrap_err(); - assert_eq!(std::mem::discriminant(&err), source_file_truncated_disc()); + assert_eq!(discriminant(&err), source_file_truncated_disc()); } #[test] @@ -1078,9 +1079,6 @@ mod tests { .write_to_file("/nonexistent/directory/rewritten.gguf") .unwrap_err(); - assert_eq!( - std::mem::discriminant(&err), - destination_file_write_failed_disc() - ); + assert_eq!(discriminant(&err), destination_file_write_failed_disc()); } } diff --git a/llama-cpp-bindings/src/grammar_matcher.rs b/llama-cpp-bindings/src/grammar_matcher.rs index cd512c13f..a4cbeb430 100644 --- a/llama-cpp-bindings/src/grammar_matcher.rs +++ b/llama-cpp-bindings/src/grammar_matcher.rs @@ -1,4 +1,5 @@ use std::any::Any; +use std::fmt::Display; use std::panic::AssertUnwindSafe; use std::panic::catch_unwind; @@ -109,7 +110,7 @@ impl GrammarMatcher { } } - fn run( + fn run( &mut self, operation: &'static str, step: impl FnOnce(&mut TokenParser) -> Result, @@ -131,6 +132,8 @@ impl GrammarMatcher { #[cfg(test)] mod tests { + use std::any::Any; + use llguidance::ParserFactory; use llguidance::TokenParser; use llguidance::api::StopReason; @@ -182,21 +185,21 @@ mod tests { #[test] fn a_static_str_panic_payload_is_preserved() { - let payload: Box = Box::new("parser exploded"); + let payload: Box = Box::new("parser exploded"); assert_eq!(panic_payload_message(payload.as_ref()), "parser exploded"); } #[test] fn an_owned_string_panic_payload_is_preserved() { - let payload: Box = Box::new("lexer exploded".to_owned()); + let payload: Box = Box::new("lexer exploded".to_owned()); assert_eq!(panic_payload_message(payload.as_ref()), "lexer exploded"); } #[test] fn a_payload_without_a_message_is_described() { - let payload: Box = Box::new(42_u32); + let payload: Box = Box::new(42_u32); assert_eq!( panic_payload_message(payload.as_ref()), diff --git a/llama-cpp-bindings/src/json_schema_to_grammar.rs b/llama-cpp-bindings/src/json_schema_to_grammar.rs index b030f9e83..2526d5303 100644 --- a/llama-cpp-bindings/src/json_schema_to_grammar.rs +++ b/llama-cpp-bindings/src/json_schema_to_grammar.rs @@ -1,8 +1,10 @@ use std::ffi::{CStr, CString, c_char}; +use std::ptr; -use crate::error::json_schema_to_grammar_error::JsonSchemaToGrammarError; use llama_cpp_ffi_status::read_and_free_cpp_string; +use crate::error::json_schema_to_grammar_error::JsonSchemaToGrammarError; + /// # Safety /// /// On `LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_OK` the function reads and frees `out` as a @@ -98,8 +100,8 @@ unsafe fn json_schema_to_grammar_status_to_result( /// the wrapper reports any non-OK status, or the returned grammar is not valid UTF-8. pub fn json_schema_to_grammar(schema_json: &str) -> Result { let schema_cstr = CString::new(schema_json)?; - let mut out: *mut c_char = std::ptr::null_mut(); - let mut error_ptr: *mut c_char = std::ptr::null_mut(); + let mut out: *mut c_char = ptr::null_mut(); + let mut error_ptr: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_json_schema_to_grammar( @@ -114,6 +116,9 @@ pub fn json_schema_to_grammar(schema_json: &str) -> Result crate::error::Result<()> { + fn mark_init() -> Result<()> { match LLAMA_BACKEND_INITIALIZED.compare_exchange(false, true, SeqCst, SeqCst) { Ok(_was_uninitialized) => Ok(()), Err(_was_already_initialized) => Err(LlamaCppError::BackendAlreadyInitialized), @@ -19,7 +20,7 @@ impl LlamaBackend { /// # Errors /// Returns an error if the backend was already initialized. - pub fn init() -> crate::error::Result { + pub fn init() -> Result { Self::mark_init()?; unsafe { llama_cpp_bindings_sys::llama_backend_init() } Ok(Self {}) @@ -30,7 +31,7 @@ impl LlamaBackend { /// /// # Errors /// Returns an error if the backend was already initialized. - pub fn init_numa(strategy: NumaStrategy) -> crate::error::Result { + pub fn init_numa(strategy: NumaStrategy) -> Result { let backend = Self::init()?; unsafe { @@ -67,10 +68,13 @@ impl Drop for LlamaBackend { #[cfg(test)] mod tests { + use std::mem::discriminant; + use serial_test::serial; use super::LlamaBackend; use crate::error::llama_cpp_error::LlamaCppError; + use crate::llama_backend_numa_strategy::NumaStrategy; #[test] #[serial] @@ -86,8 +90,8 @@ mod tests { let second_err = LlamaBackend::init().unwrap_err(); assert_eq!( - std::mem::discriminant(&second_err), - std::mem::discriminant(&LlamaCppError::BackendAlreadyInitialized), + discriminant(&second_err), + discriminant(&LlamaCppError::BackendAlreadyInitialized), "expected BackendAlreadyInitialized, got {second_err:?}" ); } @@ -95,14 +99,12 @@ mod tests { #[test] #[serial] fn init_numa_returns_error_when_backend_already_initialized() { - use crate::llama_backend_numa_strategy::NumaStrategy; - let _backend = LlamaBackend::init().unwrap(); let second_err = LlamaBackend::init_numa(NumaStrategy::Disabled).unwrap_err(); assert_eq!( - std::mem::discriminant(&second_err), - std::mem::discriminant(&LlamaCppError::BackendAlreadyInitialized), + discriminant(&second_err), + discriminant(&LlamaCppError::BackendAlreadyInitialized), "expected BackendAlreadyInitialized, got {second_err:?}" ); } @@ -128,8 +130,6 @@ mod tests { #[test] #[serial] fn init_numa_with_a_numa_strategy_initializes_the_backend_numa_support_needs() { - use crate::llama_backend_numa_strategy::NumaStrategy; - assert_eq!( LlamaBackend::init_numa(NumaStrategy::Distribute), Ok(LlamaBackend {}) diff --git a/llama-cpp-bindings/src/llama_backend_device.rs b/llama-cpp-bindings/src/llama_backend_device.rs index 2a09daee5..3b9eba3f5 100644 --- a/llama-cpp-bindings/src/llama_backend_device.rs +++ b/llama-cpp-bindings/src/llama_backend_device.rs @@ -1,13 +1,14 @@ -use crate::llama_backend_device_type::LlamaBackendDeviceType; +use std::ffi::CStr; use std::ffi::c_char; +use std::mem; + +use crate::llama_backend_device_type::LlamaBackendDeviceType; fn cstr_to_string(ptr: *const c_char) -> String { if ptr.is_null() { String::new() } else { - unsafe { std::ffi::CStr::from_ptr(ptr) } - .to_string_lossy() - .to_string() + unsafe { CStr::from_ptr(ptr) }.to_string_lossy().to_string() } } @@ -31,7 +32,7 @@ impl LlamaBackendDevice { for device_index in 0..device_count { let dev = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_get(device_index) }; let props = unsafe { - let mut props = std::mem::zeroed(); + let mut props = mem::zeroed(); llama_cpp_bindings_sys::ggml_backend_dev_get_props(dev, &raw mut props); props }; @@ -61,12 +62,16 @@ impl LlamaBackendDevice { #[cfg(test)] mod tests { + use std::ptr; + use super::cstr_to_string; use crate::llama_backend_device::LlamaBackendDevice; + #[cfg(feature = "dynamic-backends")] + use crate::load_backends::load_backends; #[test] fn cstr_to_string_with_null_returns_empty() { - let result = cstr_to_string(std::ptr::null()); + let result = cstr_to_string(ptr::null()); assert_eq!(result, ""); } @@ -81,7 +86,7 @@ mod tests { #[test] fn list_devices_returns_at_least_one() { #[cfg(feature = "dynamic-backends")] - crate::load_backends::load_backends().unwrap(); + load_backends().unwrap(); let devices = LlamaBackendDevice::list_all(); assert!(!devices.is_empty()); diff --git a/llama-cpp-bindings/src/llama_batch.rs b/llama-cpp-bindings/src/llama_batch.rs index d76b24f14..09513e2e7 100644 --- a/llama-cpp-bindings/src/llama_batch.rs +++ b/llama-cpp-bindings/src/llama_batch.rs @@ -142,6 +142,7 @@ impl LlamaBatch { mod tests { use llama_cpp_bindings_sys::llama_rs_batch_entry; + use super::LlamaBatch; use crate::batch_add_error::BatchAddError; use crate::context::context_limits::ContextLimits; use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; @@ -149,8 +150,6 @@ mod tests { use crate::sampled_token::SampledToken; use crate::token::LlamaToken; - use super::LlamaBatch; - const LIMITS: ContextLimits = ContextLimits { n_batch: 4, n_ubatch: 4, diff --git a/llama-cpp-bindings/src/llguidance_sampler.rs b/llama-cpp-bindings/src/llguidance_sampler.rs index 89994fa84..396a9fbad 100644 --- a/llama-cpp-bindings/src/llguidance_sampler.rs +++ b/llama-cpp-bindings/src/llguidance_sampler.rs @@ -1,12 +1,17 @@ +use std::ffi::c_char; use std::ffi::c_void; +use std::ptr; +use std::slice; use std::sync::Arc; -use llama_cpp_error_recorder::RecordedError; -use llama_cpp_error_recorder::record; +use llguidance::api::TopLevelGrammar; use llguidance::earley::SlicedBiasComputer; use toktrie::ApproximateTokEnv; use toktrie::InferenceCapabilities; +use llama_cpp_error_recorder::RecordedError; +use llama_cpp_error_recorder::record; + use crate::error::grammar_error::GrammarError; use crate::grammar_matcher::GrammarMatcher; use crate::mask_outcome::MaskOutcome; @@ -22,7 +27,7 @@ struct LlgContext { const unsafe extern "C" fn llg_name( _smpl: *const llama_cpp_bindings_sys::llama_sampler, -) -> *const std::os::raw::c_char { +) -> *const c_char { c"llguidance".as_ptr() } @@ -54,7 +59,7 @@ unsafe extern "C" fn llg_apply( } }; - let data = unsafe { std::slice::from_raw_parts_mut(cur_p.data, cur_p.size) }; + let data = unsafe { slice::from_raw_parts_mut(cur_p.data, cur_p.size) }; for item in data.iter_mut() { if !mask.is_allowed(item.id.cast_unsigned()) { item.logit = f32::NEG_INFINITY; @@ -140,9 +145,11 @@ fn create_llg_sampler_from_tok_env( message: factory_error.to_string(), })?; - let grammar = llguidance::api::TopLevelGrammar::from_tagged_str(grammar_kind, grammar_data) - .map_err(|parse_error| GrammarError::LlguidanceGrammarInvalid { - message: parse_error.to_string(), + let grammar = + TopLevelGrammar::from_tagged_str(grammar_kind, grammar_data).map_err(|parse_error| { + GrammarError::LlguidanceGrammarInvalid { + message: parse_error.to_string(), + } })?; let parser = factory.create_parser(grammar).map_err(|parser_error| { @@ -161,7 +168,7 @@ fn create_llg_sampler_from_tok_env( LlamaSampler::initialize("llguidance", |out_sampler, out_error| unsafe { llama_cpp_bindings_sys::llama_rs_sampler_init( &raw mut LLG_SAMPLER_I, - std::ptr::null_mut(), + ptr::null_mut(), out_sampler, out_error, ) @@ -189,11 +196,11 @@ mod tests { use toktrie::ApproximateTokEnv; use toktrie::INVALID_TOKEN; + use toktrie::InferenceCapabilities; use toktrie::TokRxInfo; use toktrie::TokTrie; use llama_cpp_fault_injection::ArmedFault; - use toktrie::InferenceCapabilities; use super::UNCOMPILABLE_BIAS_SLICE; use super::create_llg_sampler_from_tok_env; diff --git a/llama-cpp-bindings/src/load_backends_from_path.rs b/llama-cpp-bindings/src/load_backends_from_path.rs index 2ce89ea1c..6acdde9c2 100644 --- a/llama-cpp-bindings/src/load_backends_from_path.rs +++ b/llama-cpp-bindings/src/load_backends_from_path.rs @@ -22,39 +22,31 @@ pub fn load_backends_from_path(path: &Path) -> Result<(), LoadBackendsError> { #[cfg(test)] mod tests { + use std::ffi::CString; + use std::ffi::OsStr; + use std::mem::discriminant; + use std::os::unix::ffi::OsStrExt; + use std::path::PathBuf; + use super::load_backends_from_path; use crate::load_backends_error::LoadBackendsError; - use std::path::PathBuf; #[test] fn load_backends_from_path_returns_path_null_byte_for_embedded_null() { - use std::ffi::CString; - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - let path = PathBuf::from(OsStr::from_bytes(b"/tmp/foo\0bar")); let err = load_backends_from_path(&path).unwrap_err(); let representative = LoadBackendsError::PathNullByte(CString::new(b"a\0b".to_vec()).unwrap_err()); - assert_eq!( - std::mem::discriminant(&err), - std::mem::discriminant(&representative) - ); + assert_eq!(discriminant(&err), discriminant(&representative)); } #[test] fn load_backends_from_path_returns_path_not_utf8_for_invalid_utf8() { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - let path = PathBuf::from(OsStr::from_bytes(b"/tmp/\xff\xfe")); let err = load_backends_from_path(&path).unwrap_err(); let representative = LoadBackendsError::PathNotUtf8(PathBuf::new()); - assert_eq!( - std::mem::discriminant(&err), - std::mem::discriminant(&representative) - ); + assert_eq!(discriminant(&err), discriminant(&representative)); } } diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index 305a9e1a1..794e4cc40 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -16,8 +16,13 @@ pub mod tokenizer_input; pub mod vocab_type; pub mod vocab_type_from_int_error; +use std::cmp; use std::ffi::{CStr, CString, c_char}; +use std::fmt; +use std::fmt::Debug; +use std::fmt::Formatter; use std::num::NonZeroU16; +use std::num::TryFromIntError; use std::os::raw::c_int; use std::path::Path; use std::ptr; @@ -26,11 +31,14 @@ use std::sync::Arc; use encoding_rs::CoderResult; use once_cell::sync::OnceCell; +use params::LlamaModelParams; use toktrie::ApproximateTokEnv; use toktrie::INVALID_TOKEN; use toktrie::TokRxInfo; use toktrie::TokTrie; +use llama_cpp_ffi_status::read_and_free_cpp_string; + use crate::chat_templates::ChatTemplates; use crate::error::chat_template_error::ChatTemplateError; use crate::error::chat_templates_error::ChatTemplatesError; @@ -43,19 +51,16 @@ use crate::error::token_outside_vocabulary::TokenOutsideVocabulary; use crate::error::token_to_string_error::TokenToStringError; use crate::llama_backend::LlamaBackend; use crate::llama_token_attrs::LlamaTokenAttrs; -use crate::model::tokenizer_input::TokenizerInput; -use crate::sampled_token::SampledToken; -use crate::token::LlamaToken; - use crate::model::add_bos::AddBos; use crate::model::llama_chat_template::LlamaChatTemplate; use crate::model::llama_lora_adapter::LlamaLoraAdapter; use crate::model::parse_special_tokens::ParseSpecialTokens; use crate::model::rope_type::RopeType; +use crate::model::tokenizer_input::TokenizerInput; use crate::model::vocab_type::VocabType; use crate::model::vocab_type_from_int_error::VocabTypeFromIntError; -use llama_cpp_ffi_status::read_and_free_cpp_string; -use params::LlamaModelParams; +use crate::sampled_token::SampledToken; +use crate::token::LlamaToken; const PIECE_PROBE_LEN: c_int = 32; @@ -110,8 +115,8 @@ pub struct LlamaModel { chat_templates: OnceCell, } -impl std::fmt::Debug for LlamaModel { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl Debug for LlamaModel { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { f.debug_struct("LlamaModel") .field("model", &self.model) .finish_non_exhaustive() @@ -219,7 +224,7 @@ impl LlamaModel { /// # Errors /// /// Returns an error if the value returned by llama.cpp does not fit into a `u32`. - pub fn n_ctx_train(&self) -> Result { + pub fn n_ctx_train(&self) -> Result { let n_ctx_train = unsafe { llama_cpp_bindings_sys::llama_model_n_ctx_train(self.model.as_ptr()) }; @@ -307,7 +312,7 @@ impl LlamaModel { ParseSpecialTokens::Never => false, }; - let tokens_estimation = std::cmp::max(8, (str.len() / 2) + usize::from(add_bos)); + let tokens_estimation = cmp::max(8, (str.len() / 2) + usize::from(add_bos)); let TokenizerInput { text: c_string, length: c_string_len, @@ -491,21 +496,21 @@ impl LlamaModel { /// # Errors /// /// Returns an error if the layer count returned by llama.cpp does not fit into a `u32`. - pub fn n_layer(&self) -> Result { + pub fn n_layer(&self) -> Result { u32::try_from(unsafe { llama_cpp_bindings_sys::llama_model_n_layer(self.model.as_ptr()) }) } /// # Errors /// /// Returns an error if the head count returned by llama.cpp does not fit into a `u32`. - pub fn n_head(&self) -> Result { + pub fn n_head(&self) -> Result { u32::try_from(unsafe { llama_cpp_bindings_sys::llama_model_n_head(self.model.as_ptr()) }) } /// # Errors /// /// Returns an error if the KV head count returned by llama.cpp does not fit into a `u32`. - pub fn n_head_kv(&self) -> Result { + pub fn n_head_kv(&self) -> Result { u32::try_from(unsafe { llama_cpp_bindings_sys::llama_model_n_head_kv(self.model.as_ptr()) }) } @@ -955,14 +960,13 @@ mod cstring_with_validated_len_tests { #[cfg(test)] mod validate_string_length_for_tokenizer_tests { use super::validate_string_length_for_tokenizer; + use crate::error::string_to_token_error::StringToTokenError; #[test] fn validate_string_length_overflow_returns_error() { assert_eq!( validate_string_length_for_tokenizer(usize::MAX), - i32::try_from(usize::MAX).map_err( - crate::error::string_to_token_error::StringToTokenError::CIntConversionError - ) + i32::try_from(usize::MAX).map_err(StringToTokenError::CIntConversionError) ); } } @@ -1000,7 +1004,10 @@ mod checked_token_count_tests { #[cfg(test)] mod tokenize_into_buffer_tests { + use std::cell::Cell; + use super::tokenize_into_buffer; + use crate::error::string_to_token_error::StringToTokenError; #[test] fn tokenize_into_buffer_single_pass_sets_length() { @@ -1011,7 +1018,7 @@ mod tokenize_into_buffer_tests { #[test] fn tokenize_into_buffer_grows_buffer_when_first_pass_reports_negative_size() { - let call_count = std::cell::Cell::new(0); + let call_count = Cell::new(0); let buffer = tokenize_into_buffer(8, |_tokens, _n_tokens_max| { let count = call_count.get(); call_count.set(count + 1); @@ -1026,32 +1033,26 @@ mod tokenize_into_buffer_tests { #[test] fn tokenize_into_buffer_propagates_invocation_error() { let result = tokenize_into_buffer(8, |_tokens, _n_tokens_max| { - Err(crate::error::string_to_token_error::StringToTokenError::NotEnoughMemory) + Err(StringToTokenError::NotEnoughMemory) }); - assert_eq!( - result, - Err(crate::error::string_to_token_error::StringToTokenError::NotEnoughMemory) - ); + assert_eq!(result, Err(StringToTokenError::NotEnoughMemory)); } #[test] fn tokenize_into_buffer_propagates_second_invocation_error() { - let call_count = std::cell::Cell::new(0); + let call_count = Cell::new(0); let result = tokenize_into_buffer(8, |_tokens, _n_tokens_max| { let count = call_count.get(); call_count.set(count + 1); if count == 0 { Ok(-20) } else { - Err(crate::error::string_to_token_error::StringToTokenError::NotEnoughMemory) + Err(StringToTokenError::NotEnoughMemory) } }); - assert_eq!( - result, - Err(crate::error::string_to_token_error::StringToTokenError::NotEnoughMemory) - ); + assert_eq!(result, Err(StringToTokenError::NotEnoughMemory)); assert_eq!(call_count.get(), 2); } @@ -1059,11 +1060,9 @@ mod tokenize_into_buffer_tests { fn tokenize_into_buffer_refuses_a_capacity_beyond_a_c_int_before_allocating() { assert_eq!( tokenize_into_buffer(usize::MAX, |_tokens, _n_tokens_max| Ok(0)), - Err( - crate::error::string_to_token_error::StringToTokenError::CIntConversionError( - i32::try_from(usize::MAX).unwrap_err() - ) - ) + Err(StringToTokenError::CIntConversionError( + i32::try_from(usize::MAX).unwrap_err() + )) ); } @@ -1071,17 +1070,15 @@ mod tokenize_into_buffer_tests { fn tokenize_into_buffer_refuses_a_required_size_it_cannot_negate() { assert_eq!( tokenize_into_buffer(8, |_tokens, _n_tokens_max| Ok(i32::MIN)), - Err( - crate::error::string_to_token_error::StringToTokenError::CIntConversionError( - usize::try_from(i32::MIN).unwrap_err() - ) - ) + Err(StringToTokenError::CIntConversionError( + usize::try_from(i32::MIN).unwrap_err() + )) ); } #[test] fn tokenize_into_buffer_negative_final_size_returns_conversion_error() { - let call_count = std::cell::Cell::new(0); + let call_count = Cell::new(0); let result = tokenize_into_buffer(8, |_tokens, _n_tokens_max| { let count = call_count.get(); call_count.set(count + 1); @@ -1090,15 +1087,16 @@ mod tokenize_into_buffer_tests { assert_eq!( result.unwrap_err(), - crate::error::string_to_token_error::StringToTokenError::CIntConversionError( - usize::try_from(-5i32).unwrap_err() - ) + StringToTokenError::CIntConversionError(usize::try_from(-5i32).unwrap_err()) ); } } #[cfg(test)] mod extract_meta_string_tests { + use std::cell::Cell; + use std::slice; + use super::extract_meta_string; use crate::error::meta_val_error::MetaValError; @@ -1106,8 +1104,7 @@ mod extract_meta_string_tests { fn returns_error_when_null_terminator_missing() { let result = extract_meta_string( |buf_ptr, buf_len| { - let buffer = - unsafe { std::slice::from_raw_parts_mut(buf_ptr.cast::(), buf_len) }; + let buffer = unsafe { slice::from_raw_parts_mut(buf_ptr.cast::(), buf_len) }; buffer[0] = b'a'; buffer[1] = b'b'; buffer[2] = b'c'; @@ -1130,8 +1127,7 @@ mod extract_meta_string_tests { fn returns_error_for_invalid_utf8_data() { let result = extract_meta_string( |buf_ptr, buf_len| { - let buffer = - unsafe { std::slice::from_raw_parts_mut(buf_ptr.cast::(), buf_len) }; + let buffer = unsafe { slice::from_raw_parts_mut(buf_ptr.cast::(), buf_len) }; buffer[0] = 0xFF; buffer[1] = 0xFE; buffer[2] = 0; @@ -1150,7 +1146,7 @@ mod extract_meta_string_tests { let initial_capacity: usize = 4; let length_exceeding_initial_capacity = 10; let written_length = 2; - let call_count = std::cell::Cell::new(0); + let call_count = Cell::new(0); let result = extract_meta_string( |buf_ptr, buf_len| { let count = call_count.get(); @@ -1159,7 +1155,7 @@ mod extract_meta_string_tests { length_exceeding_initial_capacity } else { let buffer = - unsafe { std::slice::from_raw_parts_mut(buf_ptr.cast::(), buf_len) }; + unsafe { slice::from_raw_parts_mut(buf_ptr.cast::(), buf_len) }; buffer[0] = b'h'; buffer[1] = b'i'; buffer[2] = 0; @@ -1175,6 +1171,7 @@ mod extract_meta_string_tests { #[cfg(test)] mod initialize_lora_adapter_tests { + use std::mem::ManuallyDrop; use std::ptr; use super::initialize_lora_adapter; @@ -1190,7 +1187,7 @@ mod initialize_lora_adapter_tests { #[test] fn lora_adapter_initialization_owns_a_valid_pointer() { let pointer = ptr::NonNull::dangling(); - let adapter = std::mem::ManuallyDrop::new( + let adapter = ManuallyDrop::new( initialize_lora_adapter(|| pointer.as_ptr()) .expect("a non-null adapter pointer must initialize"), ); diff --git a/llama-cpp-bindings/src/model/llama_chat_template.rs b/llama-cpp-bindings/src/model/llama_chat_template.rs index 56ead4f14..06d79a6f3 100644 --- a/llama-cpp-bindings/src/model/llama_chat_template.rs +++ b/llama-cpp-bindings/src/model/llama_chat_template.rs @@ -1,4 +1,8 @@ +use std::ffi::NulError; use std::ffi::{CStr, CString}; +use std::fmt; +use std::fmt::Debug; +use std::fmt::Formatter; use std::str::Utf8Error; #[derive(Eq, PartialEq, Clone, PartialOrd, Ord, Hash)] @@ -7,7 +11,7 @@ pub struct LlamaChatTemplate(pub CString); impl LlamaChatTemplate { /// # Errors /// Returns an error if the template string contains null bytes. - pub fn new(template: &str) -> Result { + pub fn new(template: &str) -> Result { Ok(Self(CString::new(template)?)) } @@ -29,8 +33,8 @@ impl LlamaChatTemplate { } } -impl std::fmt::Debug for LlamaChatTemplate { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl Debug for LlamaChatTemplate { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { self.0.fmt(f) } } diff --git a/llama-cpp-bindings/src/model/llama_load_mode.rs b/llama-cpp-bindings/src/model/llama_load_mode.rs index 73618744e..41b01b44b 100644 --- a/llama-cpp-bindings/src/model/llama_load_mode.rs +++ b/llama-cpp-bindings/src/model/llama_load_mode.rs @@ -42,6 +42,7 @@ impl From for llama_cpp_bindings_sys::llama_load_mode { #[cfg(test)] mod tests { use crate::model::llama_load_mode::LlamaLoadMode; + use crate::model::llama_load_mode_parse_error::LlamaLoadModeParseError; const LOAD_MODES: &[(LlamaLoadMode, llama_cpp_bindings_sys::llama_load_mode)] = &[ ( @@ -91,11 +92,7 @@ mod tests { fn unknown_ffi_load_mode_preserves_its_value() { assert_eq!( LlamaLoadMode::try_from(i32::MAX), - Err( - crate::model::llama_load_mode_parse_error::LlamaLoadModeParseError { - value: i32::MAX - } - ) + Err(LlamaLoadModeParseError { value: i32::MAX }) ); } } diff --git a/llama-cpp-bindings/src/model/llama_lora_adapter.rs b/llama-cpp-bindings/src/model/llama_lora_adapter.rs index 99e873eda..0517a20b9 100644 --- a/llama-cpp-bindings/src/model/llama_lora_adapter.rs +++ b/llama-cpp-bindings/src/model/llama_lora_adapter.rs @@ -38,6 +38,7 @@ impl Drop for LlamaLoraAdapter<'_> { mod ownership_tests { use std::marker::PhantomData; use std::mem::ManuallyDrop; + use std::ptr; use std::ptr::NonNull; use crate::model::llama_lora_adapter::LlamaLoraAdapter; @@ -53,7 +54,7 @@ mod ownership_tests { #[test] fn dropping_an_adapter_releases_its_native_pointer() { drop(LlamaLoraAdapter { - lora_adapter: std::ptr::null_mut(), + lora_adapter: ptr::null_mut(), model: PhantomData, }); } diff --git a/llama-cpp-bindings/src/model/params.rs b/llama-cpp-bindings/src/model/params.rs index 6c963acd0..10cf93546 100644 --- a/llama-cpp-bindings/src/model/params.rs +++ b/llama-cpp-bindings/src/model/params.rs @@ -1,13 +1,18 @@ use std::ffi::{CStr, CString, c_char}; +use std::fmt; use std::fmt::{Debug, Formatter}; use std::pin::Pin; +use std::ptr; use std::ptr::null; +use llama_cpp_ffi_status::read_and_free_cpp_string; + use crate::backend_device_index::backend_device_index; use crate::context::params::LlamaContextParams; use crate::error::fit_error::FitError; use crate::error::llama_cpp_error::LlamaCppError; use crate::error::model_params_error::ModelParamsError; +use crate::max_devices::max_devices; use crate::model::llama_lazy_mode::LlamaLazyMode; use crate::model::llama_lazy_mode_parse_error::LlamaLazyModeParseError; use crate::model::llama_load_mode::LlamaLoadMode; @@ -17,7 +22,6 @@ use crate::model::params::fit_extra_model::FitExtraModel; use crate::model::params::fit_result::FitResult; use crate::model::params::kv_overrides::KvOverrides; use crate::model::split_mode::LlamaSplitMode; -use llama_cpp_ffi_status::read_and_free_cpp_string; pub mod fit_extra_model; pub mod fit_result; @@ -137,7 +141,7 @@ pub struct LlamaModelParams { } impl Debug for LlamaModelParams { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { f.debug_struct("LlamaModelParams") .field("n_gpu_layers", &self.params.n_gpu_layers) .field("main_gpu", &self.params.main_gpu) @@ -259,7 +263,7 @@ impl LlamaModelParams { self.buft_overrides .push(llama_cpp_bindings_sys::llama_model_tensor_buft_override { pattern: null(), - buft: std::ptr::null_mut(), + buft: ptr::null_mut(), }); self.params.tensor_buft_overrides = self.buft_overrides.as_ptr(); @@ -371,9 +375,9 @@ impl LlamaModelParams { /// Returns `LlamaCppError::BackendDeviceNotFound` if any device index is invalid. pub fn with_devices(mut self, devices: &[usize]) -> Result { for dev in self.devices.iter_mut() { - *dev = std::ptr::null_mut(); + *dev = ptr::null_mut(); } - let max_devices = crate::max_devices::max_devices().min(LLAMA_CPP_MAX_DEVICES); + let max_devices = max_devices().min(LLAMA_CPP_MAX_DEVICES); if devices.len() > max_devices { return Err(LlamaCppError::MaxDevicesExceeded(max_devices)); } @@ -421,7 +425,7 @@ impl LlamaModelParams { max_buft + 1, llama_cpp_bindings_sys::llama_model_tensor_buft_override { pattern: null(), - buft: std::ptr::null_mut(), + buft: ptr::null_mut(), }, ); @@ -429,7 +433,7 @@ impl LlamaModelParams { self.params.tensor_buft_overrides = null(); let mut out_unrecognized_status_code: i32 = 0; - let mut out_error: *mut c_char = std::ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); let mut extra_ffi = extra.as_mut().map(|extra_model| extra_model.as_ffi()); let extra_ptr = extra_ffi @@ -477,10 +481,10 @@ impl Default for LlamaModelParams { }], buft_overrides: vec![llama_cpp_bindings_sys::llama_model_tensor_buft_override { pattern: null(), - buft: std::ptr::null_mut(), + buft: ptr::null_mut(), }], buft_override_patterns: Vec::new(), - devices: Box::pin([std::ptr::null_mut(); 16]), + devices: Box::pin([ptr::null_mut(); 16]), tensor_split: Vec::new(), } } @@ -488,16 +492,26 @@ impl Default for LlamaModelParams { #[cfg(test)] mod tests { + use std::ffi::CStr; + use std::ffi::CString; + use std::mem::discriminant; use std::ptr; use super::LLAMA_CPP_MAX_DEVICES; use super::LlamaModelParams; use super::fit_params_status_to_result; + use crate::context::params::LlamaContextParams; use crate::error::fit_error::FitError; use crate::error::llama_cpp_error::LlamaCppError; + use crate::error::model_params_error::ModelParamsError; + use crate::llama_backend::LlamaBackend; + #[cfg(feature = "dynamic-backends")] + use crate::load_backends::load_backends; + use crate::max_devices::max_devices; use crate::model::llama_lazy_mode::LlamaLazyMode; use crate::model::llama_lazy_mode_parse_error::LlamaLazyModeParseError; use crate::model::llama_load_mode::LlamaLoadMode; + use crate::model::params::param_override_value::ParamOverrideValue; use crate::model::split_mode::LlamaSplitMode; fn contract_error_reported_for( @@ -516,7 +530,7 @@ mod tests { #[test] fn device_capacity_is_not_smaller_than_the_upstream_limit() { - let upstream_limit = crate::max_devices::max_devices(); + let upstream_limit = max_devices(); assert!( LLAMA_CPP_MAX_DEVICES >= upstream_limit, "LLAMA_CPP_MAX_DEVICES ({LLAMA_CPP_MAX_DEVICES}) backs a fixed-size device array and is \ @@ -732,12 +746,12 @@ mod tests { params .as_mut() - .add_cpu_buft_override(std::ffi::CStr::from_bytes_with_nul(&caller_buffer).unwrap()) + .add_cpu_buft_override(CStr::from_bytes_with_nul(&caller_buffer).unwrap()) .unwrap(); caller_buffer.fill(b'x'); assert_eq!( - unsafe { std::ffi::CStr::from_ptr(params.buft_overrides[0].pattern) }, + unsafe { CStr::from_ptr(params.buft_overrides[0].pattern) }, c"blk\\.0\\.ffn" ); } @@ -751,10 +765,7 @@ mod tests { .unwrap(); let result = params.as_mut().add_cpu_buft_override(c"second_pattern"); - assert_eq!( - result.unwrap_err(), - crate::error::model_params_error::ModelParamsError::SlotNotEmpty - ); + assert_eq!(result.unwrap_err(), ModelParamsError::SlotNotEmpty); } #[test] @@ -767,9 +778,6 @@ mod tests { #[test] fn append_kv_override_twice_fails_with_slot_not_empty() { - use crate::model::params::param_override_value::ParamOverrideValue; - use std::ffi::CString; - let mut params = std::pin::pin!(LlamaModelParams::default()); let key = CString::new("first_key").unwrap(); params @@ -782,17 +790,14 @@ mod tests { .as_mut() .append_kv_override(&key2, ParamOverrideValue::Int(2)); - assert_eq!( - result.unwrap_err(), - crate::error::model_params_error::ModelParamsError::SlotNotEmpty - ); + assert_eq!(result.unwrap_err(), ModelParamsError::SlotNotEmpty); } #[test] fn with_devices_too_many_returns_max_exceeded() { let too_many: Vec = (0..17).collect(); let result = LlamaModelParams::default().with_devices(&too_many); - let device_limit = crate::max_devices::max_devices().min(LLAMA_CPP_MAX_DEVICES); + let device_limit = max_devices().min(LLAMA_CPP_MAX_DEVICES); assert!( matches!( @@ -806,7 +811,7 @@ mod tests { #[test] fn with_devices_sets_devices_when_available() { #[cfg(feature = "dynamic-backends")] - crate::load_backends::load_backends().unwrap(); + load_backends().unwrap(); let dev_count = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_count() }; assert!(dev_count > 0, "Test requires at least one backend device"); @@ -833,48 +838,40 @@ mod tests { #[test] fn append_kv_override_with_high_byte_returns_invalid_character_error() { - use crate::model::params::param_override_value::ParamOverrideValue; - let key_bytes: &[u8] = b"\xff\0"; - let key = std::ffi::CStr::from_bytes_with_nul(key_bytes).unwrap(); + let key = CStr::from_bytes_with_nul(key_bytes).unwrap(); let mut params = std::pin::pin!(LlamaModelParams::default()); let result = params .as_mut() .append_kv_override(key, ParamOverrideValue::Int(1)); assert_eq!( - std::mem::discriminant(&result.unwrap_err()), - std::mem::discriminant( - &crate::error::model_params_error::ModelParamsError::InvalidCharacterInKey { - byte: 0, - reason: String::new(), - } - ), + discriminant(&result.unwrap_err()), + discriminant(&ModelParamsError::InvalidCharacterInKey { + byte: 0, + reason: String::new(), + }), ); } #[test] fn add_cpu_buft_override_with_high_byte_returns_invalid_character_error() { let key_bytes: &[u8] = b"\xff\0"; - let key = std::ffi::CStr::from_bytes_with_nul(key_bytes).unwrap(); + let key = CStr::from_bytes_with_nul(key_bytes).unwrap(); let mut params = std::pin::pin!(LlamaModelParams::default()); let result = params.as_mut().add_cpu_buft_override(key); assert_eq!( - std::mem::discriminant(&result.unwrap_err()), - std::mem::discriminant( - &crate::error::model_params_error::ModelParamsError::InvalidCharacterInKey { - byte: 0, - reason: String::new(), - } - ), + discriminant(&result.unwrap_err()), + discriminant(&ModelParamsError::InvalidCharacterInKey { + byte: 0, + reason: String::new(), + }), ); } #[test] fn append_kv_override_with_empty_slot_vector_returns_no_available_slot() { - use crate::model::params::param_override_value::ParamOverrideValue; - let mut params = LlamaModelParams::default(); params.kv_overrides.clear(); let mut pinned = std::pin::pin!(params); @@ -883,10 +880,7 @@ mod tests { .as_mut() .append_kv_override(c"any_key", ParamOverrideValue::Int(1)); - assert_eq!( - result.unwrap_err(), - crate::error::model_params_error::ModelParamsError::NoAvailableSlot - ); + assert_eq!(result.unwrap_err(), ModelParamsError::NoAvailableSlot); } #[test] @@ -897,23 +891,16 @@ mod tests { let result = pinned.as_mut().add_cpu_buft_override(c"any_pattern"); - assert_eq!( - result.unwrap_err(), - crate::error::model_params_error::ModelParamsError::NoAvailableSlot - ); + assert_eq!(result.unwrap_err(), ModelParamsError::NoAvailableSlot); } #[test] #[serial_test::serial] fn fit_params_invalid_model_path_returns_error() { - use crate::context::params::LlamaContextParams; - use crate::error::fit_error::FitError; - use crate::llama_backend::LlamaBackend; - let _backend = LlamaBackend::init(); let mut params = std::pin::pin!(LlamaModelParams::default()); let mut context_params = LlamaContextParams::default(); - let mut margins = vec![0usize; crate::max_devices::max_devices()]; + let mut margins = vec![0usize; max_devices()]; let bogus_path = c"/nonexistent/path/to/model.gguf"; let result = params.as_mut().fit_params( @@ -930,12 +917,9 @@ mod tests { #[test] fn fit_params_refuses_margins_not_sized_to_the_device_count() { - use crate::context::params::LlamaContextParams; - use crate::error::fit_error::FitError; - let mut params = std::pin::pin!(LlamaModelParams::default()); let mut context_params = LlamaContextParams::default(); - let mut margins = vec![0usize; crate::max_devices::max_devices() + 1]; + let mut margins = vec![0usize; max_devices() + 1]; let result = params.as_mut().fit_params( c"/nonexistent/path/to/model.gguf", @@ -949,8 +933,8 @@ mod tests { assert_eq!( result.err(), Some(FitError::MarginsDeviceCountMismatch { - expected: crate::max_devices::max_devices(), - actual: crate::max_devices::max_devices() + 1, + expected: max_devices(), + actual: max_devices() + 1, }) ); } diff --git a/llama-cpp-bindings/src/model/params/kv_overrides.rs b/llama-cpp-bindings/src/model/params/kv_overrides.rs index 822e27106..0d8888876 100644 --- a/llama-cpp-bindings/src/model/params/kv_overrides.rs +++ b/llama-cpp-bindings/src/model/params/kv_overrides.rs @@ -26,12 +26,13 @@ impl<'model_params> IntoIterator for KvOverrides<'model_params> { #[cfg(test)] mod tests { - use crate::model::params::kv_override_entry::KvOverrideEntry; use std::ffi::CString; use std::pin::pin; use crate::model::params::LlamaModelParams; + use crate::model::params::kv_override_entry::KvOverrideEntry; use crate::model::params::param_override_value::ParamOverrideValue; + use crate::model::params::unknown_kv_override_tag::UnknownKvOverrideTag; #[test] fn kv_overrides_empty_by_default() { @@ -91,9 +92,6 @@ mod tests { .next() .expect("one override must be present"); - assert_eq!( - entry.unwrap_err(), - crate::model::params::unknown_kv_override_tag::UnknownKvOverrideTag(9999) - ); + assert_eq!(entry.unwrap_err(), UnknownKvOverrideTag(9999)); } } diff --git a/llama-cpp-bindings/src/model/params/param_override_value.rs b/llama-cpp-bindings/src/model/params/param_override_value.rs index 041371a54..b6765ef25 100644 --- a/llama-cpp-bindings/src/model/params/param_override_value.rs +++ b/llama-cpp-bindings/src/model/params/param_override_value.rs @@ -1,3 +1,5 @@ +use std::ffi::c_char; + use crate::model::params::unknown_kv_override_tag::UnknownKvOverrideTag; #[derive(Debug, Clone, Copy, PartialEq)] @@ -5,7 +7,7 @@ pub enum ParamOverrideValue { Bool(bool), Float(f64), Int(i64), - Str([std::os::raw::c_char; 128]), + Str([c_char; 128]), } impl ParamOverrideValue { @@ -68,6 +70,8 @@ impl TryFrom<&llama_cpp_bindings_sys::llama_model_kv_override> for ParamOverride #[cfg(test)] mod tests { + use std::mem; + use super::ParamOverrideValue; #[test] @@ -220,7 +224,7 @@ mod tests { let ffi_override = llama_cpp_bindings_sys::llama_model_kv_override { key: [0; 128], tag: 9999, - __bindgen_anon_1: unsafe { std::mem::zeroed() }, + __bindgen_anon_1: unsafe { mem::zeroed() }, }; let result = ParamOverrideValue::try_from(&ffi_override); diff --git a/llama-cpp-bindings/src/mtmd/eval_chunk.rs b/llama-cpp-bindings/src/mtmd/eval_chunk.rs index a797164a5..9aaf8887c 100644 --- a/llama-cpp-bindings/src/mtmd/eval_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/eval_chunk.rs @@ -1,13 +1,13 @@ use std::ffi::c_char; +use std::ptr; use llama_cpp_ffi_status::read_and_free_cpp_string; -use crate::context::LlamaContext; -use crate::context::decoded_output_rows::DecodedOutputRows; - use super::mtmd_context::MtmdContext; use super::mtmd_eval_error::MtmdEvalError; use super::mtmd_input_chunk::MtmdInputChunk; +use crate::context::LlamaContext; +use crate::context::decoded_output_rows::DecodedOutputRows; fn eval_chunk_single_status_to_result( status: llama_cpp_bindings_sys::llama_rs_mtmd_eval_chunk_single_status, @@ -87,7 +87,7 @@ pub fn eval_chunk( ) -> Result { let mut final_position: llama_cpp_bindings_sys::llama_pos = start_position; let mut out_llama_cpp_return_code: i32 = 0; - let mut out_error: *mut c_char = std::ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_mtmd_eval_chunk_single( diff --git a/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs b/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs index f56a3324d..b6237dbee 100644 --- a/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs @@ -1,8 +1,6 @@ use std::borrow::Borrow; use std::marker::PhantomData; -use crate::context::LlamaContext; - use super::eval_chunk::eval_chunk; use super::fitted_mtmd_input_chunks::FittedMtmdInputChunks; use super::mtmd_context::MtmdContext; @@ -11,6 +9,7 @@ use super::mtmd_eval_limits::MtmdEvalLimits; use super::mtmd_input_chunk::MtmdInputChunk; use super::mtmd_input_chunk_type::MtmdInputChunkType; use super::mtmd_input_chunks::MtmdInputChunks; +use crate::context::LlamaContext; #[derive(Debug)] pub struct FittedMtmdInputChunk<'fitted> { diff --git a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs index 47b1e6bd1..56bfb6c26 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs @@ -1,5 +1,7 @@ +use std::ffi::NulError; use std::ffi::{CStr, CString, c_char}; use std::path::PathBuf; +use std::ptr; use std::ptr::NonNull; use std::slice; @@ -284,8 +286,8 @@ impl MtmdBitmap { return Err(MtmdBitmapError::InvalidDataSize); } - let mut out_bitmap: *mut llama_cpp_bindings_sys::mtmd_bitmap = std::ptr::null_mut(); - let mut out_error: *mut c_char = std::ptr::null_mut(); + let mut out_bitmap: *mut llama_cpp_bindings_sys::mtmd_bitmap = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_mtmd_bitmap_init( nx, @@ -303,8 +305,8 @@ impl MtmdBitmap { /// /// Returns the [`MtmdBitmapError`] variant matching the wrapper's status code. pub fn from_audio_data(data: &[f32]) -> Result { - let mut out_bitmap: *mut llama_cpp_bindings_sys::mtmd_bitmap = std::ptr::null_mut(); - let mut out_error: *mut c_char = std::ptr::null_mut(); + let mut out_bitmap: *mut llama_cpp_bindings_sys::mtmd_bitmap = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_mtmd_bitmap_init_from_audio( data.len(), @@ -322,8 +324,8 @@ impl MtmdBitmap { /// Returns an [`MtmdBitmapError`] variant matching the wrapper's status code. pub fn from_file(ctx: &MtmdContext, path: &str) -> Result { let path_cstr = CString::new(path)?; - let mut out_bitmap: *mut llama_cpp_bindings_sys::mtmd_bitmap = std::ptr::null_mut(); - let mut out_error: *mut c_char = std::ptr::null_mut(); + let mut out_bitmap: *mut llama_cpp_bindings_sys::mtmd_bitmap = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_mtmd_bitmap_init_from_file( @@ -341,8 +343,8 @@ impl MtmdBitmap { /// /// Returns an [`MtmdBitmapError`] variant matching the wrapper's status code. pub fn from_buffer(ctx: &MtmdContext, data: &[u8]) -> Result { - let mut out_bitmap: *mut llama_cpp_bindings_sys::mtmd_bitmap = std::ptr::null_mut(); - let mut out_error: *mut c_char = std::ptr::null_mut(); + let mut out_bitmap: *mut llama_cpp_bindings_sys::mtmd_bitmap = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_mtmd_bitmap_init_from_buf( @@ -390,7 +392,7 @@ impl MtmdBitmap { /// /// Returns an error if the ID string contains null bytes. /// - pub fn set_id(&self, id: &str) -> Result<(), std::ffi::NulError> { + pub fn set_id(&self, id: &str) -> Result<(), NulError> { let id_cstr = CString::new(id)?; unsafe { llama_cpp_bindings_sys::mtmd_bitmap_set_id(self.bitmap.as_ptr(), id_cstr.as_ptr()); @@ -524,6 +526,7 @@ mod tests { #[cfg(test)] mod cstr_ptr_to_optional_string_tests { + use std::ffi::CString; use std::ptr; use super::cstr_ptr_to_optional_string; @@ -535,7 +538,7 @@ mod cstr_ptr_to_optional_string_tests { #[test] fn cstr_ptr_to_optional_string_returns_some_for_valid() { - let cstr = std::ffi::CString::new("hello").unwrap(); + let cstr = CString::new("hello").unwrap(); let result = cstr_ptr_to_optional_string(cstr.as_ptr()); assert_eq!(result, Some("hello".to_string())); @@ -547,9 +550,10 @@ mod from_file_status_to_result_tests { use std::path::PathBuf; use std::ptr; + use llama_cpp_ffi_status::FfiStatusError; + use super::from_file_status_to_result; use crate::mtmd::mtmd_bitmap_error::MtmdBitmapError; - use llama_cpp_ffi_status::FfiStatusError; #[test] fn from_file_success_with_null_bitmap_is_contract_error() { diff --git a/llama-cpp-bindings/src/mtmd/mtmd_context.rs b/llama-cpp-bindings/src/mtmd/mtmd_context.rs index 0d9fcbc23..227f611aa 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_context.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_context.rs @@ -5,9 +5,8 @@ use std::ptr; use std::ptr::NonNull; use enumflags2::BitFlags; -use llama_cpp_ffi_status::read_and_free_cpp_string; -use crate::model::LlamaModel; +use llama_cpp_ffi_status::read_and_free_cpp_string; use super::mtmd_bitmap::MtmdBitmap; use super::mtmd_capability::MtmdCapability; @@ -20,6 +19,7 @@ use super::mtmd_input_chunks::MtmdInputChunks; use super::mtmd_input_part::MtmdInputPart; use super::mtmd_input_text::MtmdInputText; use super::mtmd_tokenize_error::MtmdTokenizeError; +use crate::model::LlamaModel; fn capability_flags( llama_cpp_bindings_sys::mtmd_helper_cap { diff --git a/llama-cpp-bindings/src/mtmd/mtmd_eval_limits.rs b/llama-cpp-bindings/src/mtmd/mtmd_eval_limits.rs index 5b88c4123..1fac6b1d8 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_eval_limits.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_eval_limits.rs @@ -1,10 +1,9 @@ use std::num::NonZeroU32; +use super::mtmd_eval_error::MtmdEvalError; use crate::context::LlamaContext; use crate::context::context_limits::ContextLimits; -use super::mtmd_eval_error::MtmdEvalError; - #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct MtmdEvalLimits { pub context: ContextLimits, diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs index ed54c311d..a67f2e8b0 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs @@ -6,14 +6,13 @@ use std::slice; use llama_cpp_ffi_status::read_and_free_cpp_string; -use crate::token::LlamaToken; - use super::mtmd_context::MtmdContext; use super::mtmd_eval_error::MtmdEvalError; use super::mtmd_input_chunk_error::MtmdInputChunkError; use super::mtmd_input_chunk_type::MtmdInputChunkType; use super::mtmd_input_chunk_type_error::MtmdInputChunkTypeError; use super::non_causal_chunk_micro_batch_mismatch::NonCausalChunkMicroBatchMismatch; +use crate::token::LlamaToken; /// # Safety /// @@ -220,6 +219,8 @@ impl Drop for MtmdInputChunk { #[cfg(test)] mod unit_tests { + use std::ptr; + use super::non_causal_chunk_micro_batch_error; use super::tokens_from_raw_ptr; use crate::mtmd::mtmd_eval_error::MtmdEvalError; @@ -227,7 +228,7 @@ mod unit_tests { #[test] fn tokens_from_raw_ptr_returns_none_for_null() { - assert!(unsafe { tokens_from_raw_ptr(std::ptr::null(), 5) }.is_none()); + assert!(unsafe { tokens_from_raw_ptr(ptr::null(), 5) }.is_none()); } #[test] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs index 7d9f85cae..7da92537f 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs @@ -4,8 +4,6 @@ use std::ptr::NonNull; use llama_cpp_ffi_status::read_and_free_cpp_string; -use crate::context::LlamaContext; - use super::eval_chunk::eval_chunk; use super::mtmd_context::MtmdContext; use super::mtmd_eval_error::MtmdEvalError; @@ -13,6 +11,7 @@ use super::mtmd_eval_limits::MtmdEvalLimits; use super::mtmd_input_chunk::MtmdInputChunk; use super::mtmd_input_chunk_type::MtmdInputChunkType; use super::mtmd_input_chunks_error::MtmdInputChunksError; +use crate::context::LlamaContext; fn input_chunks_init_status_to_result( status: llama_cpp_bindings_sys::llama_rs_mtmd_input_chunks_init_status, diff --git a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs index 703b1d72d..456b9e0b0 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs @@ -5,11 +5,10 @@ use std::ptr; use llama_cpp_ffi_status::read_and_free_cpp_string; -use crate::backend_device_index::backend_device_index; - use super::mtmd_context_params::MtmdContextParams; use super::mtmd_image_token_limit::MtmdImageTokenLimit; use super::mtmd_memory_usage_error::MtmdMemoryUsageError; +use crate::backend_device_index::backend_device_index; const OPERATION: &str = "llama_rs_mtmd_get_memory_usage"; diff --git a/llama-cpp-bindings/src/sampled_token_classifier.rs b/llama-cpp-bindings/src/sampled_token_classifier.rs index f86261d05..6cf21432f 100644 --- a/llama-cpp-bindings/src/sampled_token_classifier.rs +++ b/llama-cpp-bindings/src/sampled_token_classifier.rs @@ -1,6 +1,5 @@ use llama_cpp_bindings_sys::llama_pos; use llama_cpp_bindings_sys::llama_seq_id; - use llama_cpp_bindings_types::TokenUsage; use llama_cpp_bindings_types::TokenUsageError; diff --git a/llama-cpp-bindings/src/sampling.rs b/llama-cpp-bindings/src/sampling.rs index b6fa18ff8..5c657fc29 100644 --- a/llama-cpp-bindings/src/sampling.rs +++ b/llama-cpp-bindings/src/sampling.rs @@ -1,10 +1,16 @@ use std::borrow::Borrow; use std::ffi::{CString, c_char}; +use std::fmt; use std::fmt::{Debug, Formatter}; +use std::mem; +use std::ptr; use std::ptr::NonNull; use llama_cpp_error_recorder::ErrorScope; use llama_cpp_error_recorder::RecordedError; +use llama_cpp_ffi_status::read_and_free_cpp_string; +use llama_cpp_gbnf::gbnf_validation_error::GbnfValidationError; +use llama_cpp_gbnf::validate_gbnf::validate_gbnf; use crate::context::LlamaContext; use crate::dry_sequence_breakers::dry_sequence_breakers; @@ -13,14 +19,12 @@ use crate::error::sample_error::SampleError; use crate::error::sampler_accept_error::SamplerAcceptError; use crate::error::sampler_apply_error::SamplerApplyError; use crate::error::sampling_error::SamplingError; +use crate::llguidance_sampler::create_llg_sampler; use crate::model::LlamaModel; use crate::sanitized_grammar::SanitizedGrammar; use crate::token::LlamaToken; use crate::token::data_array::LlamaTokenDataArray; use crate::token::logit_bias::LlamaLogitBias; -use llama_cpp_ffi_status::read_and_free_cpp_string; -use llama_cpp_gbnf::gbnf_validation_error::GbnfValidationError; -use llama_cpp_gbnf::validate_gbnf::validate_gbnf; fn check_sampler_accept_status( status: llama_cpp_bindings_sys::llama_rs_sampler_accept_status, @@ -68,7 +72,7 @@ fn check_sampler_accept_status( fn sampler_apply_status_to_result( status: llama_cpp_bindings_sys::llama_rs_sampler_apply_status, - out_error: *mut std::os::raw::c_char, + out_error: *mut c_char, ) -> Result<(), SamplerApplyError> { match status { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_OK => Ok(()), @@ -407,7 +411,7 @@ fn grammar_callback_error_to_accept_result( } impl Debug for LlamaSampler { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { f.debug_struct("LlamaSampler").finish_non_exhaustive() } } @@ -437,15 +441,15 @@ impl LlamaSampler { *mut *mut c_char, ) -> llama_cpp_bindings_sys::llama_rs_sampler_init_status, ) -> Result { - let mut sampler: *mut llama_cpp_bindings_sys::llama_sampler = std::ptr::null_mut(); - let mut error_ptr: *mut c_char = std::ptr::null_mut(); + let mut sampler: *mut llama_cpp_bindings_sys::llama_sampler = ptr::null_mut(); + let mut error_ptr: *mut c_char = ptr::null_mut(); let status = initialize_sampler(&raw mut sampler, &raw mut error_ptr); sampler_init_status_to_result(status, sampler, error_ptr, sampler_name) } fn take_into_chain(&self, sampler: Self) -> Result<(), SamplingError> { - let mut error_ptr: *mut c_char = std::ptr::null_mut(); + let mut error_ptr: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_sampler_chain_add( self.sampler.as_ptr(), @@ -454,7 +458,7 @@ impl LlamaSampler { ) }; - sampler_chain_add_status_to_result(status, error_ptr).map(|()| std::mem::forget(sampler)) + sampler_chain_add_status_to_result(status, error_ptr).map(|()| mem::forget(sampler)) } #[must_use] @@ -471,7 +475,7 @@ impl LlamaSampler { ctx.get_logits_ith(idx)?; let mut token: i32 = -1; - let mut error_ptr: *mut c_char = std::ptr::null_mut(); + let mut error_ptr: *mut c_char = ptr::null_mut(); let scope = ErrorScope::enter(); let status = unsafe { @@ -509,7 +513,7 @@ impl LlamaSampler { ) -> Result<(), SamplerApplyError> { unsafe { data_array.modify_as_c_llama_token_data_array(|c_llama_token_data_array| { - let mut out_error: *mut c_char = std::ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); let status = llama_cpp_bindings_sys::llama_rs_sampler_apply( self.sampler.as_ptr(), c_llama_token_data_array, @@ -530,7 +534,7 @@ impl LlamaSampler { /// # Errors /// Returns [`SamplerAcceptError`] if the underlying sampler rejects the token. pub fn accept(&mut self, token: LlamaToken) -> Result<(), SamplerAcceptError> { - let mut error_ptr: *mut c_char = std::ptr::null_mut(); + let mut error_ptr: *mut c_char = ptr::null_mut(); let scope = ErrorScope::enter(); let status = unsafe { @@ -715,8 +719,8 @@ impl LlamaSampler { grammar: grammar_cstring, root: root_cstring, } = Self::sanitize_grammar_strings(grammar_str, grammar_root)?; - let mut sampler: *mut llama_cpp_bindings_sys::llama_sampler = std::ptr::null_mut(); - let mut error_ptr: *mut c_char = std::ptr::null_mut(); + let mut sampler: *mut llama_cpp_bindings_sys::llama_sampler = ptr::null_mut(); + let mut error_ptr: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_sampler_init_grammar( @@ -747,8 +751,8 @@ impl LlamaSampler { root: root_cstring, } = Self::sanitize_grammar_strings(grammar_str, grammar_root)?; let trigger_patterns = Self::sanitize_trigger_patterns(trigger_patterns)?; - let mut sampler: *mut llama_cpp_bindings_sys::llama_sampler = std::ptr::null_mut(); - let mut error_ptr: *mut c_char = std::ptr::null_mut(); + let mut sampler: *mut llama_cpp_bindings_sys::llama_sampler = ptr::null_mut(); + let mut error_ptr: *mut c_char = ptr::null_mut(); let mut trigger_pattern_ptrs: Vec<*const c_char> = trigger_patterns.iter().map(|cs| cs.as_ptr()).collect(); @@ -782,7 +786,7 @@ impl LlamaSampler { grammar_kind: &str, grammar_data: &str, ) -> Result { - crate::llguidance_sampler::create_llg_sampler(model, grammar_kind, grammar_data) + create_llg_sampler(model, grammar_kind, grammar_data) } fn sanitize_grammar_strings( @@ -972,6 +976,7 @@ impl Drop for LlamaSampler { mod tests { use std::ffi::CString; use std::ptr; + use std::slice; use llama_cpp_error_recorder::RecordedError; use llama_cpp_error_recorder::record; @@ -979,7 +984,13 @@ mod tests { use super::LlamaSampler; use crate::error::grammar_error::GrammarError; use crate::error::sample_error::SampleError; + use crate::error::sampler_apply_error::SamplerApplyError; + use crate::error::sampling_error::SamplingError; use crate::sanitized_grammar::SanitizedGrammar; + use crate::token::LlamaToken; + use crate::token::data::LlamaTokenData; + use crate::token::data_array::LlamaTokenDataArray; + use crate::token::logit_bias::LlamaLogitBias; unsafe extern "C" fn record_a_failure_and_exclude_every_candidate( _sampler: *mut llama_cpp_bindings_sys::llama_sampler, @@ -988,8 +999,7 @@ mod tests { record(RecordedError::new("mask failed".to_string())); let candidates = unsafe { &mut *candidates }; - let candidate_data = - unsafe { std::slice::from_raw_parts_mut(candidates.data, candidates.size) }; + let candidate_data = unsafe { slice::from_raw_parts_mut(candidates.data, candidates.size) }; for candidate in candidate_data { candidate.logit = f32::NEG_INFINITY; } @@ -1034,10 +1044,6 @@ mod tests { #[test] fn a_grammar_callback_failure_takes_precedence_over_a_failed_application() { - use crate::token::LlamaToken; - use crate::token::data::LlamaTokenData; - use crate::token::data_array::LlamaTokenDataArray; - let mut data_array = LlamaTokenDataArray::new( vec![ LlamaTokenData::new(LlamaToken::new(0), 1.0, 0.0), @@ -1060,7 +1066,7 @@ mod tests { assert_eq!( result.unwrap_err(), - crate::error::sampling_error::SamplingError::SamplerUnavailable { sampler: "test" } + SamplingError::SamplerUnavailable { sampler: "test" } ); } @@ -1140,10 +1146,6 @@ mod tests { #[test] fn apply_modifies_data_array() { - use crate::token::LlamaToken; - use crate::token::data::LlamaTokenData; - use crate::token::data_array::LlamaTokenDataArray; - let sampler = greedy_sampler(); let mut data_array = LlamaTokenDataArray::new( vec![ @@ -1160,12 +1162,6 @@ mod tests { #[test] fn apply_surfaces_a_sampler_that_excludes_every_candidate() { - use crate::error::sampler_apply_error::SamplerApplyError; - use crate::token::LlamaToken; - use crate::token::data::LlamaTokenData; - use crate::token::data_array::LlamaTokenDataArray; - use crate::token::logit_bias::LlamaLogitBias; - let excluding_sampler = LlamaSampler::logit_bias( 2, &[ @@ -1196,7 +1192,7 @@ mod tests { .expect("sampler chain must initialize"); sampler - .accept(crate::token::LlamaToken::new(1)) + .accept(LlamaToken::new(1)) .expect("test: accept should succeed"); } @@ -1205,15 +1201,13 @@ mod tests { let mut sampler = LlamaSampler::chain([penalties_sampler(), greedy_sampler()]) .expect("sampler chain must initialize"); - let result = sampler.accept(crate::token::LlamaToken::new(42)); + let result = sampler.accept(LlamaToken::new(42)); assert!(result.is_ok()); } #[test] fn accept_many_multiple_tokens() { - use crate::token::LlamaToken; - let mut sampler = LlamaSampler::chain([penalties_sampler(), greedy_sampler()]) .expect("sampler chain must initialize"); @@ -1224,9 +1218,6 @@ mod tests { #[test] fn all_sampler_constructors() { - use crate::token::LlamaToken; - use crate::token::logit_bias::LlamaLogitBias; - let _temp = LlamaSampler::temp(0.8).expect("temperature sampler must initialize"); let _temp_ext = LlamaSampler::temp_ext(0.8, 0.1, 1.0) .expect("extended temperature sampler must initialize"); @@ -1351,11 +1342,12 @@ mod grammar_callback_error_to_accept_result_tests { mod check_sampler_accept_status_tests { use std::ptr; - use super::check_sampler_accept_status; - use crate::error::sampler_accept_error::SamplerAcceptError; use llama_cpp_ffi_status::FfiContractError; use llama_cpp_ffi_status::FfiStatusError; + use super::check_sampler_accept_status; + use crate::error::sampler_accept_error::SamplerAcceptError; + #[test] fn check_sampler_accept_status_ok() { let result = check_sampler_accept_status( @@ -1470,9 +1462,10 @@ mod check_sampler_accept_status_tests { mod sampler_sample_status_to_result_tests { use std::ptr; + use llama_cpp_ffi_status::FfiStatusError; + use super::sampler_sample_status_to_result; use crate::error::sample_error::SampleError; - use llama_cpp_ffi_status::FfiStatusError; #[test] fn sampler_sample_status_allocation_failure_maps_to_not_enough_memory() { @@ -1627,6 +1620,7 @@ mod sampler_init_status_to_result_tests { use std::ptr; use super::sampler_init_status_to_result; + use crate::error::sampling_error::SamplingError; #[test] fn sampler_init_status_null_maps_to_sampler_unavailable() { @@ -1638,11 +1632,9 @@ mod sampler_init_status_to_result_tests { "penalties", ) .err(), - Some( - crate::error::sampling_error::SamplingError::SamplerUnavailable { - sampler: "penalties" - } - ) + Some(SamplingError::SamplerUnavailable { + sampler: "penalties" + }) ); } @@ -1656,7 +1648,7 @@ mod sampler_init_status_to_result_tests { "penalties", ) .err(), - Some(crate::error::sampling_error::SamplingError::NotEnoughMemory) + Some(SamplingError::NotEnoughMemory) ); } @@ -1670,11 +1662,9 @@ mod sampler_init_status_to_result_tests { "penalties", ) .err(), - Some( - crate::error::sampling_error::SamplingError::LlamaCppOutOfMemory { - sampler: "penalties" - } - ) + Some(SamplingError::LlamaCppOutOfMemory { + sampler: "penalties" + }) ); } @@ -1691,7 +1681,7 @@ mod sampler_init_status_to_result_tests { "penalties", ) .err(), - Some(crate::error::sampling_error::SamplingError::Reported { + Some(SamplingError::Reported { sampler: "penalties", message: "vector too long".to_owned(), }) @@ -1787,6 +1777,7 @@ mod sampler_chain_add_status_to_result_tests { use std::ptr; use super::sampler_chain_add_status_to_result; + use crate::error::sampling_error::SamplingError; #[test] fn sampler_chain_add_status_allocation_failure_maps_to_not_enough_memory() { @@ -1795,7 +1786,7 @@ mod sampler_chain_add_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_ERROR_STRING_ALLOCATION_FAILED, ptr::null_mut(), ), - Err(crate::error::sampling_error::SamplingError::NotEnoughMemory) + Err(SamplingError::NotEnoughMemory) ); } @@ -1806,11 +1797,7 @@ mod sampler_chain_add_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), ), - Err( - crate::error::sampling_error::SamplingError::LlamaCppOutOfMemory { - sampler: "chain" - } - ) + Err(SamplingError::LlamaCppOutOfMemory { sampler: "chain" }) ); } @@ -1824,7 +1811,7 @@ mod sampler_chain_add_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION, out_error, ), - Err(crate::error::sampling_error::SamplingError::Reported { + Err(SamplingError::Reported { sampler: "chain", message: "vector too long".to_owned(), }) @@ -1915,9 +1902,10 @@ mod sampler_chain_add_status_to_result_tests { mod sampler_init_grammar_status_to_result_tests { use std::ptr; + use llama_cpp_ffi_status::FfiStatusError; + use super::sampler_init_grammar_status_to_result; use crate::error::grammar_error::GrammarError; - use llama_cpp_ffi_status::FfiStatusError; #[test] fn sampler_init_grammar_status_null_maps_to_grammar_malformed() { @@ -2046,9 +2034,10 @@ mod sampler_init_grammar_status_to_result_tests { mod sampler_init_grammar_lazy_patterns_status_to_result_tests { use std::ptr; + use llama_cpp_ffi_status::FfiStatusError; + use super::sampler_init_grammar_lazy_patterns_status_to_result; use crate::error::grammar_error::GrammarError; - use llama_cpp_ffi_status::FfiStatusError; #[test] fn sampler_init_grammar_lazy_patterns_status_null_maps_to_lazy_patterns_grammar_malformed() { @@ -2193,6 +2182,8 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { #[cfg(test)] mod sampler_application_tests { + use std::ptr; + use llama_cpp_fault_injection::ArmedFault; use llama_cpp_fault_injection::injected_fault_message; @@ -2243,7 +2234,7 @@ mod sampler_application_tests { LlamaSampler::initialize("overreporting", |out_sampler, out_error| unsafe { llama_cpp_bindings_sys::llama_rs_sampler_init( &raw mut OVERREPORTING_SAMPLER, - std::ptr::null_mut(), + ptr::null_mut(), out_sampler, out_error, ) diff --git a/llama-cpp-bindings/src/send_logs_to_log.rs b/llama-cpp-bindings/src/send_logs_to_log.rs index 6a91b2e57..cf1153d52 100644 --- a/llama-cpp-bindings/src/send_logs_to_log.rs +++ b/llama-cpp-bindings/src/send_logs_to_log.rs @@ -8,6 +8,10 @@ ) )] +use std::ffi::CStr; +use std::ffi::c_char; +use std::ffi::c_void; +use std::sync::PoisonError; use std::sync::{Mutex, OnceLock}; use llama_cpp_log_decoder::decode_anomaly::DecodeAnomaly; @@ -108,8 +112,8 @@ fn dispatch_anomaly(source: &LogSource, anomaly: DecodeAnomaly) { unsafe extern "C" fn logs_to_log( raw_level: llama_cpp_bindings_sys::ggml_log_level, - text_ptr: *const std::os::raw::c_char, - data_ptr: *mut std::os::raw::c_void, + text_ptr: *const c_char, + data_ptr: *mut c_void, ) { let source: &LogSource = unsafe { &*data_ptr.cast::() }; @@ -125,7 +129,7 @@ unsafe extern "C" fn logs_to_log( return; } - let text_cstr = unsafe { std::ffi::CStr::from_ptr(text_ptr) }; + let text_cstr = unsafe { CStr::from_ptr(text_ptr) }; let text = text_cstr.to_string_lossy(); let incoming = ggml_level_to_incoming(raw_level); @@ -134,7 +138,7 @@ unsafe extern "C" fn logs_to_log( let mut decoder = source .decoder .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); + .unwrap_or_else(PoisonError::into_inner); decoder.feed(incoming, &text) }; @@ -153,26 +157,31 @@ pub fn send_logs_to_log(options: LogOptions) { unsafe { llama_cpp_bindings_sys::llama_log_set( Some(logs_to_log), - llama_source.cast::().cast_mut(), + llama_source.cast::().cast_mut(), ); llama_cpp_bindings_sys::ggml_log_set( Some(logs_to_log), - ggml_source.cast::().cast_mut(), + ggml_source.cast::().cast_mut(), ); } } #[cfg(test)] mod tests { - use crate::log_record::LogRecord; + use std::ffi::CStr; + use std::ffi::c_void; + use std::ptr; + use std::sync::PoisonError; use std::sync::{Mutex, Once}; + use std::thread; + + use log::{Level, Log, Metadata, Record}; + use serial_test::serial; use llama_cpp_log_decoder::decode_output::DecodeOutput; use llama_cpp_log_decoder::incoming_log_level::IncomingLogLevel; use llama_cpp_log_decoder::log_level::LogLevel; use llama_cpp_log_decoder::log_line::LogLine; - use log::{Level, Log, Metadata, Record}; - use serial_test::serial; use super::GGML_SOURCE; use super::LLAMA_SOURCE; @@ -182,6 +191,7 @@ mod tests { use super::logs_to_log; use super::resolve_record; use crate::log_options::LogOptions; + use crate::log_record::LogRecord; use crate::send_logs_to_log::send_logs_to_log; #[derive(Debug)] @@ -201,10 +211,7 @@ mod tests { } fn log(&self, record: &Record) { - let mut guard = self - .records - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut guard = self.records.lock().unwrap_or_else(PoisonError::into_inner); guard.push(CapturedRecord { level: record.level(), target: record.target().to_owned(), @@ -231,7 +238,7 @@ mod tests { let guard = TEST_LOGGER .records .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); + .unwrap_or_else(PoisonError::into_inner); guard .iter() .filter(|record| record.target == target) @@ -244,12 +251,10 @@ mod tests { fn invoke_callback( level: llama_cpp_bindings_sys::ggml_log_level, - text: &std::ffi::CStr, + text: &CStr, source: &LogSource, ) { - let ptr = std::ptr::from_ref(source) - .cast::() - .cast_mut(); + let ptr = ptr::from_ref(source).cast::().cast_mut(); unsafe { logs_to_log(level, text.as_ptr(), ptr); } @@ -488,13 +493,11 @@ mod tests { let target = "test-null-text-pointer"; let source = LogSource::new(target, LogOptions::default()); - let source_ptr = std::ptr::from_ref(&source) - .cast::() - .cast_mut(); + let source_ptr = ptr::from_ref(&source).cast::().cast_mut(); unsafe { logs_to_log( llama_cpp_bindings_sys::GGML_LOG_LEVEL_INFO, - std::ptr::null(), + ptr::null(), source_ptr, ); } @@ -515,12 +518,12 @@ mod tests { let target = "test-decoder-mutex-poison"; let source = LogSource::new(target, LogOptions::default()); - std::thread::scope(|scope| { + thread::scope(|scope| { let handle = scope.spawn(|| { let _guard = source .decoder .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); + .unwrap_or_else(PoisonError::into_inner); panic!("intentional poison"); }); let _ = handle.join(); diff --git a/llama-cpp-bindings/src/token.rs b/llama-cpp-bindings/src/token.rs index 4b87459ee..8af2c903d 100644 --- a/llama-cpp-bindings/src/token.rs +++ b/llama-cpp-bindings/src/token.rs @@ -1,5 +1,7 @@ +use std::fmt; use std::fmt::Debug; use std::fmt::Display; +use std::fmt::Formatter; pub mod data; pub mod data_array; @@ -10,7 +12,7 @@ pub mod logit_bias; pub struct LlamaToken(pub llama_cpp_bindings_sys::llama_token); impl Display for LlamaToken { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { write!(f, "{}", self.0) } } diff --git a/llama-cpp-bindings/src/token/data_array.rs b/llama-cpp-bindings/src/token/data_array.rs index c80753dab..5a5240099 100644 --- a/llama-cpp-bindings/src/token/data_array.rs +++ b/llama-cpp-bindings/src/token/data_array.rs @@ -1,8 +1,7 @@ use std::ptr; -use crate::token::data::LlamaTokenData; - use super::LlamaToken; +use crate::token::data::LlamaTokenData; #[derive(Debug, Clone, PartialEq)] pub struct LlamaTokenDataArray { diff --git a/llama-cpp-bindings/src/tool_call_format.rs b/llama-cpp-bindings/src/tool_call_format.rs index 9e67f55c3..9aabba696 100644 --- a/llama-cpp-bindings/src/tool_call_format.rs +++ b/llama-cpp-bindings/src/tool_call_format.rs @@ -46,6 +46,8 @@ pub fn try_parse(body: &str, markers: &ToolCallMarkers) -> ToolCallFormatOutcome #[cfg(test)] mod tests { + use serde_json::json; + use llama_cpp_bindings_types::BracketedJsonShape; use llama_cpp_bindings_types::KeyValueXmlTagsShape; use llama_cpp_bindings_types::PairedQuoteShape; @@ -56,10 +58,10 @@ mod tests { use llama_cpp_bindings_types::ToolCallMarkers; use llama_cpp_bindings_types::ToolCallValueQuote; use llama_cpp_bindings_types::XmlTagsShape; - use serde_json::json; use super::ToolCallFormatOutcome; use super::try_parse; + use crate::chat_template_tool_calls::known_marker_candidates::known_marker_candidates; use crate::error::bracketed_args_failure::BracketedArgsFailure; use crate::error::tool_call_format_failure::ToolCallFormatFailure; @@ -234,8 +236,6 @@ mod tests { #[test] fn try_parse_returns_no_match_for_plain_content_under_every_known_shape() { - use crate::chat_template_tool_calls::known_marker_candidates::known_marker_candidates; - let plain_content = "Sorry, I cannot help with that request."; for candidate in known_marker_candidates() { @@ -249,10 +249,6 @@ mod tests { #[test] fn duck_type_resolves_qwen_xml_input_via_xml_tags_shape_first() { - use llama_cpp_bindings_types::ToolCallArguments; - - use crate::chat_template_tool_calls::known_marker_candidates::known_marker_candidates; - let qwen_input = "\n\ \n\ \n\ @@ -292,10 +288,6 @@ mod tests { #[test] fn duck_type_resolves_glm_input_via_key_value_xml_tags_shape() { - use llama_cpp_bindings_types::ToolCallArguments; - - use crate::chat_template_tool_calls::known_marker_candidates::known_marker_candidates; - let glm_input = "get_weather\ location\ Paris\ @@ -331,10 +323,6 @@ mod tests { #[test] fn duck_type_resolves_mistral_input_via_bracketed_json_shape() { - use llama_cpp_bindings_types::ToolCallArguments; - - use crate::chat_template_tool_calls::known_marker_candidates::known_marker_candidates; - let mistral_input = r#"[TOOL_CALLS]get_weather[ARGS]{"location":"Paris"}"#; let mut resolved = None; @@ -366,10 +354,6 @@ mod tests { #[test] fn duck_type_resolves_gemma_input_via_paired_quote_shape() { - use llama_cpp_bindings_types::ToolCallArguments; - - use crate::chat_template_tool_calls::known_marker_candidates::known_marker_candidates; - let gemma_input = "<|tool_call>call:get_weather{location:<|\"|>Paris<|\"|>}"; let mut resolved = None; diff --git a/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs b/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs index 66e5e9132..4b85a4921 100644 --- a/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs +++ b/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs @@ -98,12 +98,15 @@ pub fn parse( #[cfg(test)] mod tests { + use std::mem::discriminant; + + use serde_json::json; + use llama_cpp_bindings_types::BracketedJsonShape; use llama_cpp_bindings_types::ParsedToolCall; use llama_cpp_bindings_types::ToolCallArgsShape; use llama_cpp_bindings_types::ToolCallArguments; use llama_cpp_bindings_types::ToolCallMarkers; - use serde_json::json; use super::parse; use crate::error::bracketed_args_failure::BracketedArgsFailure; @@ -191,8 +194,8 @@ mod tests { let failure = result.expect_err("malformed JSON must produce a typed failure"); assert_eq!( - std::mem::discriminant(&failure), - std::mem::discriminant(&BracketedArgsFailure::InvalidJsonArguments { + discriminant(&failure), + discriminant(&BracketedArgsFailure::InvalidJsonArguments { tool_name: String::new(), message: String::new(), }), diff --git a/llama-cpp-bindings/src/tool_call_format/json_object.rs b/llama-cpp-bindings/src/tool_call_format/json_object.rs index 3755a249f..5c9ab8d4a 100644 --- a/llama-cpp-bindings/src/tool_call_format/json_object.rs +++ b/llama-cpp-bindings/src/tool_call_format/json_object.rs @@ -79,10 +79,11 @@ pub fn parse( #[cfg(test)] mod tests { + use serde_json::json; + use llama_cpp_bindings_types::JsonObjectShape; use llama_cpp_bindings_types::ParsedToolCall; use llama_cpp_bindings_types::ToolCallArguments; - use serde_json::json; use super::parse; use crate::error::json_object_failure::JsonObjectFailure; diff --git a/llama-cpp-bindings/src/tool_call_format/key_value_xml_tags.rs b/llama-cpp-bindings/src/tool_call_format/key_value_xml_tags.rs index e1fc927dc..c5d635f39 100644 --- a/llama-cpp-bindings/src/tool_call_format/key_value_xml_tags.rs +++ b/llama-cpp-bindings/src/tool_call_format/key_value_xml_tags.rs @@ -1,11 +1,12 @@ +use nom::IResult; +use nom::Parser; +use nom::bytes::complete::take_until; + use llama_cpp_bindings_types::KeyValueXmlTagsShape; use llama_cpp_bindings_types::ParsedToolCall; use llama_cpp_bindings_types::ParsedToolCallId; use llama_cpp_bindings_types::ToolCallArguments; use llama_cpp_bindings_types::ToolCallMarkers; -use nom::IResult; -use nom::Parser; -use nom::bytes::complete::take_until; use crate::error::key_value_xml_tags_failure::KeyValueXmlTagsFailure; use crate::tool_call_format::parse_step::ParseStep; @@ -168,12 +169,13 @@ pub fn parse( #[cfg(test)] mod tests { + use serde_json::json; + use llama_cpp_bindings_types::KeyValueXmlTagsShape; use llama_cpp_bindings_types::ParsedToolCall; use llama_cpp_bindings_types::ToolCallArgsShape; use llama_cpp_bindings_types::ToolCallArguments; use llama_cpp_bindings_types::ToolCallMarkers; - use serde_json::json; use super::parse; use crate::error::key_value_xml_tags_failure::KeyValueXmlTagsFailure; diff --git a/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs b/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs index 1c89f6072..b8fdbf856 100644 --- a/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs +++ b/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs @@ -206,6 +206,8 @@ pub fn parse( #[cfg(test)] mod tests { + use serde_json::json; + use llama_cpp_bindings_types::PairedQuoteShape; use llama_cpp_bindings_types::ParsedToolCall; use llama_cpp_bindings_types::ParsedToolCallId; @@ -213,7 +215,6 @@ mod tests { use llama_cpp_bindings_types::ToolCallArguments; use llama_cpp_bindings_types::ToolCallMarkers; use llama_cpp_bindings_types::ToolCallValueQuote; - use serde_json::json; use super::parse; use crate::error::paired_quote_failure::PairedQuoteFailure; diff --git a/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs b/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs index c4b822a4c..ce5d10295 100644 --- a/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs +++ b/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs @@ -1,10 +1,11 @@ +use nom::IResult; +use nom::Parser; +use nom::bytes::complete::take_until; + use llama_cpp_bindings_types::ParsedToolCall; use llama_cpp_bindings_types::ParsedToolCallId; use llama_cpp_bindings_types::ToolCallArguments; use llama_cpp_bindings_types::XmlTagsShape; -use nom::IResult; -use nom::Parser; -use nom::bytes::complete::take_until; use crate::error::xml_function_tags_failure::XmlFunctionTagsFailure; use crate::tool_call_format::scalar_value_to_json::scalar_value_to_json; @@ -174,10 +175,11 @@ pub fn parse( #[cfg(test)] mod tests { + use serde_json::json; + use llama_cpp_bindings_types::ParsedToolCall; use llama_cpp_bindings_types::ToolCallArguments; use llama_cpp_bindings_types::XmlTagsShape; - use serde_json::json; use super::parse; use crate::error::xml_function_tags_failure::XmlFunctionTagsFailure; diff --git a/llama-cpp-bindings/tests/sampler_chain_without_backend.rs b/llama-cpp-bindings/tests/sampler_chain_without_backend.rs index 4e05bd387..4dce40654 100644 --- a/llama-cpp-bindings/tests/sampler_chain_without_backend.rs +++ b/llama-cpp-bindings/tests/sampler_chain_without_backend.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings::token::LlamaToken; use llama_cpp_bindings::token::data::LlamaTokenData; diff --git a/llama-cpp-gbnf/src/validate_gbnf.rs b/llama-cpp-gbnf/src/validate_gbnf.rs index 1e38c9c4a..2c8fcd992 100644 --- a/llama-cpp-gbnf/src/validate_gbnf.rs +++ b/llama-cpp-gbnf/src/validate_gbnf.rs @@ -1,10 +1,6 @@ use std::ffi::{CString, c_char}; use std::ptr; -use llama_cpp_ffi_status::FfiContractError; -use llama_cpp_ffi_status::FfiStatusError; -use llama_cpp_ffi_status::read_and_free_cpp_string; - use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_EMPTY_RULE_SET; use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_OK; use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_ROOT_SYMBOL_MISSING; @@ -14,6 +10,9 @@ use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_UNSUPPORTED_GRAMMAR; use llama_cpp_bindings_sys::llama_rs_gbnf_validation_status; use llama_cpp_bindings_sys::llama_rs_validate_gbnf; use llama_cpp_bindings_sys::llama_vocab; +use llama_cpp_ffi_status::FfiContractError; +use llama_cpp_ffi_status::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string; use crate::gbnf_validation_error::GbnfValidationError; @@ -107,14 +106,15 @@ pub fn validate_gbnf( mod tests { use std::ffi::CString; use std::fmt::Write; + use std::ptr; + use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_THREW_CXX_EXCEPTION; use llama_cpp_ffi_status::FfiContractError; use llama_cpp_ffi_status::FfiStatusError; use super::validate_gbnf; use super::validation_status_to_result; use crate::gbnf_validation_error::GbnfValidationError; - use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_THREW_CXX_EXCEPTION; #[test] fn valid_grammar_is_accepted() { @@ -202,7 +202,7 @@ mod tests { validation_status_to_result( LLAMA_RS_GBNF_VALIDATION_THREW_CXX_EXCEPTION, "root", - std::ptr::null_mut(), + ptr::null_mut(), ), Err(GbnfValidationError::FfiContract(FfiContractError { operation: "llama_rs_validate_gbnf", @@ -236,7 +236,7 @@ mod tests { validation_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_NULL_GRAMMAR_ARG, "root", - std::ptr::null_mut(), + ptr::null_mut(), ), Err(GbnfValidationError::FfiContract(FfiContractError { operation: "llama_rs_validate_gbnf", @@ -251,7 +251,7 @@ mod tests { validation_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_NULL_ROOT_ARG, "root", - std::ptr::null_mut(), + ptr::null_mut(), ), Err(GbnfValidationError::FfiContract(FfiContractError { operation: "llama_rs_validate_gbnf", @@ -266,7 +266,7 @@ mod tests { validation_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_NULL_OUT_ERROR_ARG, "root", - std::ptr::null_mut(), + ptr::null_mut(), ), Err(GbnfValidationError::FfiContract(FfiContractError { operation: "llama_rs_validate_gbnf", @@ -281,7 +281,7 @@ mod tests { validation_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_LLAMA_CPP_OUT_OF_MEMORY, "root", - std::ptr::null_mut(), + ptr::null_mut(), ), Err(GbnfValidationError::LlamaCppOutOfMemory) ); @@ -293,7 +293,7 @@ mod tests { validation_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_ERROR_STRING_ALLOCATION_FAILED, "root", - std::ptr::null_mut(), + ptr::null_mut(), ), Err(GbnfValidationError::NotEnoughMemory) ); @@ -302,7 +302,7 @@ mod tests { #[test] fn unknown_status_is_preserved() { assert_eq!( - validation_status_to_result(255, "root", std::ptr::null_mut(),), + validation_status_to_result(255, "root", ptr::null_mut(),), Err(GbnfValidationError::FfiStatus(FfiStatusError { operation: "llama_rs_validate_gbnf", code: 255, diff --git a/llama-cpp-test-harness-macros/src/parsed_args.rs b/llama-cpp-test-harness-macros/src/parsed_args.rs index 7b7c8ba7b..3d9b6c8bc 100644 --- a/llama-cpp-test-harness-macros/src/parsed_args.rs +++ b/llama-cpp-test-harness-macros/src/parsed_args.rs @@ -263,6 +263,7 @@ mod tests { use syn::parse_str; use super::ParsedArgs; + use crate::parsed_load_mode::ParsedLoadMode; use crate::parsed_source::ParsedSource; const ALL_REQUIRED: &str = "\ @@ -289,10 +290,7 @@ mod tests { }, ); assert_eq!(parsed.model_load_params.n_gpu_layers, 0); - assert_eq!( - parsed.model_load_params.load_mode, - crate::parsed_load_mode::ParsedLoadMode::Mmap - ); + assert_eq!(parsed.model_load_params.load_mode, ParsedLoadMode::Mmap); assert_eq!(parsed.context_params.n_ctx, 512); assert_eq!(parsed.context_params.n_batch, 128); assert_eq!(parsed.context_params.n_ubatch, 64); diff --git a/llama-cpp-test-harness-macros/src/parsed_source.rs b/llama-cpp-test-harness-macros/src/parsed_source.rs index fd3451806..49d9c1ed1 100644 --- a/llama-cpp-test-harness-macros/src/parsed_source.rs +++ b/llama-cpp-test-harness-macros/src/parsed_source.rs @@ -1,3 +1,5 @@ +use std::path::Path; + use syn::Expr; use syn::ExprCall; use syn::ExprLit; @@ -58,7 +60,7 @@ impl ParsedSource { pub fn display_suffix(&self) -> String { match self { Self::HuggingFace { file, .. } => file.clone(), - Self::LocalPath(path) => std::path::Path::new(path) + Self::LocalPath(path) => Path::new(path) .file_name() .and_then(|name| name.to_str()) .map_or_else(|| path.clone(), str::to_owned), diff --git a/llama-cpp-test-harness/src/download_model.rs b/llama-cpp-test-harness/src/download_model.rs index 2fb844fe6..80271612a 100644 --- a/llama-cpp-test-harness/src/download_model.rs +++ b/llama-cpp-test-harness/src/download_model.rs @@ -1,13 +1,14 @@ use std::path::PathBuf; use anyhow::Result; +use hf_hub::api::sync::ApiBuilder; /// # Errors /// /// Returns an error if the HF client cannot be built or the file cannot be downloaded /// (e.g., the repo or file does not exist, or network access fails). pub fn download_model(repo: &str, file: &str) -> Result { - Ok(hf_hub::api::sync::ApiBuilder::from_env() + Ok(ApiBuilder::from_env() .with_progress(true) .build() .and_then(|api| api.model(repo.to_owned()).get(file))?) diff --git a/llama-cpp-test-harness/src/execution_phase.rs b/llama-cpp-test-harness/src/execution_phase.rs index ed81fa6f6..be607f872 100644 --- a/llama-cpp-test-harness/src/execution_phase.rs +++ b/llama-cpp-test-harness/src/execution_phase.rs @@ -26,13 +26,12 @@ impl ExecutionPhase { #[cfg(test)] mod tests { + use super::ExecutionPhase; use crate::GgufSource; use crate::LlamaLoadMode; use crate::load_key::LoadKey; use crate::model_load_params::ModelLoadParams; - use super::ExecutionPhase; - fn phase_with_source(source: GgufSource) -> ExecutionPhase { ExecutionPhase { key: LoadKey { diff --git a/llama-cpp-test-harness/src/execution_plan.rs b/llama-cpp-test-harness/src/execution_plan.rs index 75d0d32e8..e52a23c7d 100644 --- a/llama-cpp-test-harness/src/execution_plan.rs +++ b/llama-cpp-test-harness/src/execution_plan.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use libtest_mimic::Arguments; use libtest_mimic::Conclusion; + use llama_cpp_bindings::llama_backend::LlamaBackend; use crate::execution_phase::ExecutionPhase; @@ -73,6 +74,7 @@ impl ExecutionPlan { #[cfg(test)] mod tests { + use super::ExecutionPlan; use crate::LlamaLoadMode; use crate::context_params::ContextParams; use crate::gguf_source::GgufSource; @@ -82,8 +84,6 @@ mod tests { use crate::model_load_params::ModelLoadParams; use crate::no_op::no_op; - use super::ExecutionPlan; - const TRIVIAL_CONTEXT_PARAMS: ContextParams = ContextParams { n_ctx: 1, n_batch: 1, diff --git a/llama-cpp-test-harness/src/harness_run_error.rs b/llama-cpp-test-harness/src/harness_run_error.rs index bd251072c..b4d4d9e7e 100644 --- a/llama-cpp-test-harness/src/harness_run_error.rs +++ b/llama-cpp-test-harness/src/harness_run_error.rs @@ -1,6 +1,7 @@ -use llama_cpp_bindings::error::llama_cpp_error::LlamaCppError; use thiserror::Error; +use llama_cpp_bindings::error::llama_cpp_error::LlamaCppError; + use crate::harness_arguments_error::HarnessArgumentsError; #[derive(Debug, Eq, Error, PartialEq)] diff --git a/llama-cpp-test-harness/src/lib.rs b/llama-cpp-test-harness/src/lib.rs index 7b59d0bbe..bbef25d9e 100644 --- a/llama-cpp-test-harness/src/lib.rs +++ b/llama-cpp-test-harness/src/lib.rs @@ -27,6 +27,12 @@ pub mod scheduled_phase; #[cfg(test)] mod test_backend_gate; +#[doc(hidden)] +pub use inventory; + +pub use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; +pub use llama_cpp_test_harness_macros::llama_test; + pub use crate::context_params::ContextParams; pub use crate::execution_phase::ExecutionPhase; pub use crate::execution_plan::ExecutionPlan; @@ -43,8 +49,3 @@ pub use crate::phase_state::PhaseState; pub use crate::run::run; pub use crate::run_to_conclusions::run_to_conclusions; pub use crate::scheduled_phase::ScheduledPhase; -pub use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; -pub use llama_cpp_test_harness_macros::llama_test; - -#[doc(hidden)] -pub use inventory; diff --git a/llama-cpp-test-harness/src/llama_fixture.rs b/llama-cpp-test-harness/src/llama_fixture.rs index 150899b8f..5e0598d12 100644 --- a/llama-cpp-test-harness/src/llama_fixture.rs +++ b/llama-cpp-test-harness/src/llama_fixture.rs @@ -1,6 +1,7 @@ use std::path::Path; use anyhow::Result; + use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::llama_backend::LlamaBackend; use llama_cpp_bindings::model::LlamaModel; diff --git a/llama-cpp-test-harness/src/load_key.rs b/llama-cpp-test-harness/src/load_key.rs index 2e4473fef..049358f2c 100644 --- a/llama-cpp-test-harness/src/load_key.rs +++ b/llama-cpp-test-harness/src/load_key.rs @@ -32,11 +32,10 @@ mod tests { use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; + use super::LoadKey; use crate::gguf_source::GgufSource; use crate::model_load_params::ModelLoadParams; - use super::LoadKey; - fn baseline() -> LoadKey { LoadKey { model_source: GgufSource::HuggingFace { diff --git a/llama-cpp-test-harness/src/parse_harness_arguments.rs b/llama-cpp-test-harness/src/parse_harness_arguments.rs index 8f060f7a1..bf761c830 100644 --- a/llama-cpp-test-harness/src/parse_harness_arguments.rs +++ b/llama-cpp-test-harness/src/parse_harness_arguments.rs @@ -25,9 +25,8 @@ pub fn parse_harness_arguments(arguments: Arguments) -> Result LoadKey { diff --git a/llama-cpp-test-harness/src/run.rs b/llama-cpp-test-harness/src/run.rs index 89b20f213..5a9564648 100644 --- a/llama-cpp-test-harness/src/run.rs +++ b/llama-cpp-test-harness/src/run.rs @@ -27,19 +27,20 @@ pub fn run() -> ExitCode { #[cfg(test)] mod tests { use std::process::ExitCode; + use std::sync::PoisonError; use libtest_mimic::Arguments; use libtest_mimic::Conclusion; + use llama_cpp_bindings::error::llama_cpp_error::LlamaCppError; use llama_cpp_bindings::llama_backend::LlamaBackend; + use super::aggregate_exit_code; + use super::run; use crate::harness_run_error::HarnessRunError; use crate::run_to_conclusions::run_to_conclusions; use crate::test_backend_gate::BACKEND_INIT_GATE; - use super::aggregate_exit_code; - use super::run; - fn passing_conclusion() -> Conclusion { Conclusion { num_filtered_out: 0, @@ -95,7 +96,7 @@ mod tests { fn run_to_conclusions_errors_when_backend_init_fails() { let _gate = BACKEND_INIT_GATE .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); + .unwrap_or_else(PoisonError::into_inner); let _hold = LlamaBackend::init().expect("first init must succeed"); let outcome = run_to_conclusions(Arguments::default()); @@ -112,7 +113,7 @@ mod tests { fn run_returns_exit_code_two_when_backend_init_fails() { let _gate = BACKEND_INIT_GATE .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); + .unwrap_or_else(PoisonError::into_inner); let _hold = LlamaBackend::init().expect("first init must succeed"); let code = run(); diff --git a/llama-cpp-test-harness/src/run_to_conclusions.rs b/llama-cpp-test-harness/src/run_to_conclusions.rs index fed81c05e..8d2e3cf4f 100644 --- a/llama-cpp-test-harness/src/run_to_conclusions.rs +++ b/llama-cpp-test-harness/src/run_to_conclusions.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use libtest_mimic::Arguments; use libtest_mimic::Conclusion; + use llama_cpp_bindings::llama_backend::LlamaBackend; use llama_cpp_bindings::log_options::LogOptions; use llama_cpp_bindings::send_logs_to_log::send_logs_to_log; @@ -28,14 +29,15 @@ pub fn run_to_conclusions(arguments: Arguments) -> Result, Harne #[cfg(test)] mod tests { + use std::sync::PoisonError; + use libtest_mimic::Arguments; + use super::run_to_conclusions; use crate::harness_arguments_error::HarnessArgumentsError; use crate::harness_run_error::HarnessRunError; use crate::test_backend_gate::BACKEND_INIT_GATE; - use super::run_to_conclusions; - #[test] fn conflicting_arguments_stop_the_run_before_the_backend_starts() { assert_eq!( @@ -54,7 +56,7 @@ mod tests { fn empty_inventory_yields_no_conclusions_and_skips_void_logs() { let _gate = BACKEND_INIT_GATE .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); + .unwrap_or_else(PoisonError::into_inner); let conclusions = run_to_conclusions(Arguments::default()) .expect("empty inventory must run without a setup failure"); diff --git a/llama-cpp-test-harness/src/scheduled_phase.rs b/llama-cpp-test-harness/src/scheduled_phase.rs index b233482cb..9247fdaa2 100644 --- a/llama-cpp-test-harness/src/scheduled_phase.rs +++ b/llama-cpp-test-harness/src/scheduled_phase.rs @@ -4,6 +4,7 @@ use libtest_mimic::Arguments; use libtest_mimic::Conclusion; use libtest_mimic::Failed; use libtest_mimic::Trial; + use llama_cpp_bindings::llama_backend::LlamaBackend; use crate::execution_phase::ExecutionPhase; @@ -91,9 +92,11 @@ mod tests { use std::sync::PoisonError; use libtest_mimic::Arguments; + use llama_cpp_bindings::llama_backend::LlamaBackend; use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; + use super::ScheduledPhase; use crate::context_params::ContextParams; use crate::execution_phase::ExecutionPhase; use crate::gguf_source::GgufSource; @@ -104,8 +107,6 @@ mod tests { use crate::no_op::no_op; use crate::test_backend_gate::BACKEND_INIT_GATE; - use super::ScheduledPhase; - static REGISTRATION: LlamaTestRegistration = LlamaTestRegistration { name: "scheduled_trial", key: LoadKey { diff --git a/llama-cpp-test-harness/src/test_backend_gate.rs b/llama-cpp-test-harness/src/test_backend_gate.rs index c762fece8..4809dabd5 100644 --- a/llama-cpp-test-harness/src/test_backend_gate.rs +++ b/llama-cpp-test-harness/src/test_backend_gate.rs @@ -1,2 +1,4 @@ +use std::sync::Mutex; + #[cfg(test)] -pub static BACKEND_INIT_GATE: std::sync::Mutex<()> = std::sync::Mutex::new(()); +pub static BACKEND_INIT_GATE: Mutex<()> = Mutex::new(()); diff --git a/llama-cpp-test-harness/tests/harness_self_test.rs b/llama-cpp-test-harness/tests/harness_self_test.rs index 9f07334da..d0cec11b6 100644 --- a/llama-cpp-test-harness/tests/harness_self_test.rs +++ b/llama-cpp-test-harness/tests/harness_self_test.rs @@ -3,6 +3,7 @@ use std::process::ExitCode; use anyhow::Result; use anyhow::bail; use libtest_mimic::Arguments; + use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; use llama_cpp_test_harness::no_op; diff --git a/llama-cpp-wrapper-sources/tests/emit_compile_commands.rs b/llama-cpp-wrapper-sources/tests/emit_compile_commands.rs index 726f7c085..f4c729008 100644 --- a/llama-cpp-wrapper-sources/tests/emit_compile_commands.rs +++ b/llama-cpp-wrapper-sources/tests/emit_compile_commands.rs @@ -1,6 +1,7 @@ use std::env; use std::fs; use std::path::PathBuf; +use std::process; use std::process::Command; use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; @@ -18,7 +19,7 @@ fn emitter_succeeded(arguments: &[&str]) -> bool { fn temporary_path(suffix: &str) -> PathBuf { env::temp_dir().join(format!( "llama-cpp-wrapper-sources-{}-{suffix}", - std::process::id() + process::id() )) } From a976862d5d9848f9dd73456893c374c75e2bb407 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sat, 31 Oct 2026 05:18:11 +0100 Subject: [PATCH 26/82] Check for a context's memory module in one place and import C types from std::ffi --- llama-cpp-bindings/src/context.rs | 11 ++++++++-- llama-cpp-bindings/src/context/memory.rs | 20 +++++++------------ llama-cpp-bindings/src/error/decode_error.rs | 2 +- llama-cpp-bindings/src/error/encode_error.rs | 2 +- .../src/error/token_to_string_error.rs | 2 +- llama-cpp-bindings/src/model.rs | 2 +- .../src/model/tokenizer_input.rs | 2 +- .../src/mtmd/mtmd_default_marker.rs | 4 ++-- 8 files changed, 23 insertions(+), 22 deletions(-) diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index 698cc0f10..9a47bb51c 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -383,6 +383,14 @@ where } } +fn memory_of( + context: NonNull, +) -> Option { + let memory = unsafe { llama_cpp_bindings_sys::llama_get_memory(context.as_ptr()) }; + + (!memory.is_null()).then_some(memory) +} + unsafe fn logits_slice<'logits>( logits: NonNull, n_vocab: i32, @@ -438,8 +446,7 @@ impl<'model> LlamaContext<'model> { let pooling_type = LlamaPoolingType::from(unsafe { llama_cpp_bindings_sys::llama_pooling_type(context.as_ptr()) }); - let keeps_no_memory = - unsafe { llama_cpp_bindings_sys::llama_get_memory(context.as_ptr()) }.is_null(); + let keeps_no_memory = memory_of(context).is_none(); let attends_causally = unsafe { llama_cpp_bindings_sys::llama_get_causal_attn(context.as_ptr()) }; diff --git a/llama-cpp-bindings/src/context/memory.rs b/llama-cpp-bindings/src/context/memory.rs index 552d92a4e..9c49cdbec 100644 --- a/llama-cpp-bindings/src/context/memory.rs +++ b/llama-cpp-bindings/src/context/memory.rs @@ -1,11 +1,12 @@ +use std::ffi::c_char; use std::ffi::c_int; use std::num::NonZeroU8; -use std::os::raw::c_char; use std::ptr; use llama_cpp_ffi_status::read_and_free_cpp_string; use crate::context::LlamaContext; +use crate::context::memory_of; use crate::error::memory_seq_add_error::MemorySeqAddError; use crate::error::memory_seq_cp_error::MemorySeqCpError; use crate::error::memory_seq_div_error::MemorySeqDivError; @@ -172,22 +173,15 @@ fn memory_seq_pos_max_status_to_result( } impl LlamaContext<'_> { - fn memory_handle(&self) -> Option { - let mem = unsafe { llama_cpp_bindings_sys::llama_get_memory(self.context.as_ptr()) }; - - if mem.is_null() { None } else { Some(mem) } - } - #[must_use] pub fn has_memory(&self) -> bool { - self.memory_handle().is_some() + memory_of(self.context).is_some() } fn required_memory_handle( &self, ) -> Result { - self.memory_handle() - .ok_or(MemorySeqCpError::MemoryHandleUnavailable) + memory_of(self.context).ok_or(MemorySeqCpError::MemoryHandleUnavailable) } /// # Errors @@ -230,7 +224,7 @@ impl LlamaContext<'_> { let p1 = p1 .map_or(Ok(-1), i32::try_from) .map_err(MemorySeqRmError::P1TooLarge)?; - let Some(mem) = self.memory_handle() else { + let Some(mem) = memory_of(self.context) else { return Ok(()); }; @@ -242,7 +236,7 @@ impl LlamaContext<'_> { } pub fn memory_clear(&mut self) { - let Some(mem) = self.memory_handle() else { + let Some(mem) = memory_of(self.context) else { return; }; let clear_data_buffers = true; @@ -250,7 +244,7 @@ impl LlamaContext<'_> { } pub fn memory_seq_keep(&mut self, seq_id: i32) { - let Some(mem) = self.memory_handle() else { + let Some(mem) = memory_of(self.context) else { return; }; unsafe { llama_cpp_bindings_sys::llama_memory_seq_keep(mem, seq_id) }; diff --git a/llama-cpp-bindings/src/error/decode_error.rs b/llama-cpp-bindings/src/error/decode_error.rs index 0f17fd1e5..1d213892f 100644 --- a/llama-cpp-bindings/src/error/decode_error.rs +++ b/llama-cpp-bindings/src/error/decode_error.rs @@ -1,5 +1,5 @@ +use std::ffi::c_int; use std::num::NonZeroI32; -use std::os::raw::c_int; use crate::error::batch_transfer_error::BatchTransferError; use crate::error::process_failure::ProcessFailure; diff --git a/llama-cpp-bindings/src/error/encode_error.rs b/llama-cpp-bindings/src/error/encode_error.rs index 4da67ef98..2c868d13c 100644 --- a/llama-cpp-bindings/src/error/encode_error.rs +++ b/llama-cpp-bindings/src/error/encode_error.rs @@ -1,5 +1,5 @@ +use std::ffi::c_int; use std::num::NonZeroI32; -use std::os::raw::c_int; use crate::error::batch_transfer_error::BatchTransferError; use crate::error::process_failure::ProcessFailure; diff --git a/llama-cpp-bindings/src/error/token_to_string_error.rs b/llama-cpp-bindings/src/error/token_to_string_error.rs index 85c12645d..7072618ea 100644 --- a/llama-cpp-bindings/src/error/token_to_string_error.rs +++ b/llama-cpp-bindings/src/error/token_to_string_error.rs @@ -1,4 +1,4 @@ -use std::os::raw::c_int; +use std::ffi::c_int; use std::string::FromUtf8Error; use crate::error::token_outside_vocabulary::TokenOutsideVocabulary; diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index 794e4cc40..6e8a942e9 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -17,13 +17,13 @@ pub mod vocab_type; pub mod vocab_type_from_int_error; use std::cmp; +use std::ffi::c_int; use std::ffi::{CStr, CString, c_char}; use std::fmt; use std::fmt::Debug; use std::fmt::Formatter; use std::num::NonZeroU16; use std::num::TryFromIntError; -use std::os::raw::c_int; use std::path::Path; use std::ptr; use std::ptr::NonNull; diff --git a/llama-cpp-bindings/src/model/tokenizer_input.rs b/llama-cpp-bindings/src/model/tokenizer_input.rs index 7953a796c..8e0b23d49 100644 --- a/llama-cpp-bindings/src/model/tokenizer_input.rs +++ b/llama-cpp-bindings/src/model/tokenizer_input.rs @@ -1,5 +1,5 @@ use std::ffi::CString; -use std::os::raw::c_int; +use std::ffi::c_int; #[derive(Debug, Eq, PartialEq)] pub struct TokenizerInput { diff --git a/llama-cpp-bindings/src/mtmd/mtmd_default_marker.rs b/llama-cpp-bindings/src/mtmd/mtmd_default_marker.rs index 553c91622..2ae3bb94b 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_default_marker.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_default_marker.rs @@ -1,5 +1,5 @@ use std::ffi::CStr; -use std::os::raw::c_char; +use std::ffi::c_char; use crate::mtmd::mtmd_default_marker_error::MtmdDefaultMarkerError; @@ -21,7 +21,7 @@ pub fn mtmd_default_marker() -> Result<&'static str, MtmdDefaultMarkerError> { #[cfg(test)] mod tests { - use std::os::raw::c_char; + use std::ffi::c_char; use std::str; use super::marker_bytes_to_str; From 82afcfe45328a5be79c67d1ec9ea708a2f308763 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sat, 31 Oct 2026 05:35:10 +0100 Subject: [PATCH 27/82] Read token pieces through one probe, evaluate mtmd chunks fetched once, report exceptions thrown while applying a sampler, and name tests after what they verify --- llama-cpp-bindings-sys/wrapper_common.cpp | 1 + .../wrapper_fault_injection.h | 1 + .../tests/memory_sequence_edits.rs | 2 +- .../tests/state_snapshots.rs | 6 +- llama-cpp-bindings/src/model.rs | 36 +++----- llama-cpp-bindings/src/model/rope_type.rs | 68 +++++---------- .../src/mtmd/mtmd_input_chunks.rs | 82 +++++++++++-------- llama-cpp-bindings/src/sampling.rs | 14 ++++ 8 files changed, 101 insertions(+), 109 deletions(-) diff --git a/llama-cpp-bindings-sys/wrapper_common.cpp b/llama-cpp-bindings-sys/wrapper_common.cpp index 50468c6ec..a32abfe94 100644 --- a/llama-cpp-bindings-sys/wrapper_common.cpp +++ b/llama-cpp-bindings-sys/wrapper_common.cpp @@ -501,6 +501,7 @@ extern "C" auto llama_rs_sampler_apply( return LLAMA_RS_SAMPLER_APPLY_NULL_OUT_ERROR_ARG; } try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_SAMPLER_APPLY_THROWS); llama_sampler_apply(sampler, data_array); return LLAMA_RS_SAMPLER_APPLY_OK; } catch (...) { diff --git a/llama-cpp-bindings-sys/wrapper_fault_injection.h b/llama-cpp-bindings-sys/wrapper_fault_injection.h index 7a13381ea..71b144958 100644 --- a/llama-cpp-bindings-sys/wrapper_fault_injection.h +++ b/llama-cpp-bindings-sys/wrapper_fault_injection.h @@ -32,6 +32,7 @@ typedef enum llama_rs_fault_point { LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_NAME_NOT_UTF8, LLAMA_RS_FAULT_POINT_REGEX_ESCAPE_THROWS, LLAMA_RS_FAULT_POINT_SAMPLER_ACCEPT_THROWS, + LLAMA_RS_FAULT_POINT_SAMPLER_APPLY_THROWS, LLAMA_RS_FAULT_POINT_SAMPLER_INIT_DIST_THROWS, LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GRAMMAR_RETURNS_NULL, LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GRAMMAR_THROWS, diff --git a/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs b/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs index 7dc303da0..3688a25c8 100644 --- a/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs +++ b/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs @@ -79,7 +79,7 @@ fn memory_seq_rm_accepts_an_explicit_range(fixture: &LlamaFixture<'_>) -> Result n_batch = 512, n_ubatch = 128, )] -fn memory_seq_cp_succeeds(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_cp_copies_a_whole_primed_sequence(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; prime_memory(fixture, &mut context)?; diff --git a/llama-cpp-bindings-tests/tests/state_snapshots.rs b/llama-cpp-bindings-tests/tests/state_snapshots.rs index e5179e658..b4a10982a 100644 --- a/llama-cpp-bindings-tests/tests/state_snapshots.rs +++ b/llama-cpp-bindings-tests/tests/state_snapshots.rs @@ -41,7 +41,7 @@ use llama_cpp_test_harness::llama_test; n_batch = 512, n_ubatch = 128, )] -fn state_get_size_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { +fn a_fresh_context_reports_a_nonempty_state_size(fixture: &LlamaFixture<'_>) -> Result<()> { let context = fixture.build_context()?; assert!(context.state_get_size() > 0); @@ -115,7 +115,9 @@ fn state_set_data_rejects_a_truncated_snapshot(fixture: &LlamaFixture<'_>) -> Re n_batch = 512, n_ubatch = 128, )] -fn state_get_data_and_state_set_data_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> { +fn state_set_data_consumes_exactly_the_bytes_state_get_data_wrote( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let mut context = fixture.build_context()?; let tokens = diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index 6e8a942e9..ddcdbdff1 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -361,23 +361,7 @@ impl LlamaModel { | SampledToken::Reasoning(inner) | SampledToken::ToolCall(inner) | SampledToken::Undeterminable(inner)) = *token; - let token = self.require_vocabulary_token(inner)?; - let mut probe = [0_u8; PIECE_PROBE_SIZE]; - - match self.write_piece(token, &mut probe, PIECE_PROBE_LEN, special, lstrip) { - Ok(written) => Ok(decode_piece(decoder, probe.split_at(written).0)), - Err(TokenToStringError::InsufficientBufferSpace(required_size)) => { - let bytes = self.token_to_piece_bytes( - token, - required_size.unsigned_abs() as usize, - special, - lstrip, - )?; - - Ok(decode_piece(decoder, &bytes)) - } - Err(probe_error) => Err(probe_error), - } + self.read_piece(inner, special, lstrip, |piece| decode_piece(decoder, piece)) } /// # Errors @@ -431,21 +415,27 @@ impl LlamaModel { } } - fn token_to_piece_bytes_fitted( + fn read_piece( &self, token: LlamaToken, special: bool, lstrip: Option, - ) -> Result, TokenToStringError> { - match self.token_to_piece_bytes(token, PIECE_PROBE_SIZE, special, lstrip) { + read: impl FnOnce(&[u8]) -> TOutput, + ) -> Result { + let token = self.require_vocabulary_token(token)?; + let mut probe = [0_u8; PIECE_PROBE_SIZE]; + + match self.write_piece(token, &mut probe, PIECE_PROBE_LEN, special, lstrip) { + Ok(written) => Ok(read(probe.split_at(written).0)), Err(TokenToStringError::InsufficientBufferSpace(required_size)) => self .token_to_piece_bytes( token, required_size.unsigned_abs() as usize, special, lstrip, - ), - probe_result => probe_result, + ) + .map(|piece| read(&piece)), + Err(probe_error) => Err(probe_error), } } @@ -704,7 +694,7 @@ fn token_piece_bytes_for_tok_env( token: LlamaToken, special: bool, ) -> Result, TokenToStringError> { - match model.token_to_piece_bytes_fitted(token, special, None) { + match model.read_piece(token, special, None, <[u8]>::to_vec) { Err(TokenToStringError::UnknownTokenType) => Ok(Vec::new()), piece_result => piece_result, } diff --git a/llama-cpp-bindings/src/model/rope_type.rs b/llama-cpp-bindings/src/model/rope_type.rs index e027c6ca7..2a970918b 100644 --- a/llama-cpp-bindings/src/model/rope_type.rs +++ b/llama-cpp-bindings/src/model/rope_type.rs @@ -26,55 +26,27 @@ mod tests { use crate::model::rope_type::RopeType; #[test] - fn rope_type_none() { + fn llama_cpp_rope_types_map_to_their_variants_and_the_rest_to_none() { assert_eq!( - RopeType::from_raw(llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NONE), - None + [ + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NORM, + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NEOX, + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_MROPE, + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_IMROPE, + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_VISION, + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NONE, + 9999, + ] + .map(RopeType::from_raw), + [ + Some(RopeType::Norm), + Some(RopeType::NeoX), + Some(RopeType::MRope), + Some(RopeType::IMRope), + Some(RopeType::Vision), + None, + None, + ] ); } - - #[test] - fn rope_type_norm() { - assert_eq!( - RopeType::from_raw(llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NORM), - Some(RopeType::Norm) - ); - } - - #[test] - fn rope_type_neox() { - assert_eq!( - RopeType::from_raw(llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NEOX), - Some(RopeType::NeoX) - ); - } - - #[test] - fn rope_type_mrope() { - assert_eq!( - RopeType::from_raw(llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_MROPE), - Some(RopeType::MRope) - ); - } - - #[test] - fn rope_type_interleaved_mrope() { - assert_eq!( - RopeType::from_raw(llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_IMROPE), - Some(RopeType::IMRope) - ); - } - - #[test] - fn rope_type_vision() { - assert_eq!( - RopeType::from_raw(llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_VISION), - Some(RopeType::Vision) - ); - } - - #[test] - fn rope_type_unknown_returns_none() { - assert_eq!(RopeType::from_raw(9999), None); - } } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs index 7da92537f..1e1c92e70 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs @@ -62,6 +62,17 @@ fn input_chunks_init_status_to_result( } } +fn fit_each( + chunks: &[MtmdInputChunk], + mtmd_ctx: &MtmdContext, + micro_batch_tokens: u32, +) -> Result, MtmdEvalError> { + chunks + .iter() + .map(|chunk| chunk.fit_to_micro_batch(mtmd_ctx, micro_batch_tokens)) + .collect() +} + #[derive(Debug)] pub struct MtmdInputChunks { pub chunks: NonNull, @@ -126,12 +137,7 @@ impl MtmdInputChunks { mtmd_ctx: &MtmdContext, micro_batch_tokens: u32, ) -> Result, MtmdEvalError> { - (0..self.len()) - .map(|index| { - self.chunk_within_count(index)? - .fit_to_micro_batch(mtmd_ctx, micro_batch_tokens) - }) - .collect() + fit_each(&self.chunks_within_count()?, mtmd_ctx, micro_batch_tokens) } #[must_use] @@ -161,37 +167,43 @@ impl MtmdInputChunks { logits_last: bool, ) -> Result { let limits = MtmdEvalLimits::of(llama_ctx); - - self.fit_to_micro_batch( - mtmd_ctx, - limits - .context - .micro_batch_tokens(limits.batch_tokens(n_batch, seq_id)?), - )?; - - let chunk_count = self.len(); - - (0..chunk_count).try_fold(start_position, |position, index| { - eval_chunk( - &self.chunk_within_count(index)?, - mtmd_ctx, - llama_ctx, - position, - seq_id, - n_batch, - logits_last && index + 1 == chunk_count, - ) - }) + let micro_batch_tokens = limits + .context + .micro_batch_tokens(limits.batch_tokens(n_batch, seq_id)?); + let chunks = self.chunks_within_count()?; + + fit_each(&chunks, mtmd_ctx, micro_batch_tokens)?; + + let chunk_count = chunks.len(); + + chunks + .iter() + .enumerate() + .try_fold(start_position, |position, (index, chunk)| { + eval_chunk( + chunk, + mtmd_ctx, + llama_ctx, + position, + seq_id, + n_batch, + logits_last && index + 1 == chunk_count, + ) + }) } - fn chunk_within_count(&self, index: usize) -> Result { - self.get(index).ok_or_else(|| { - llama_cpp_ffi_status::FfiContractError { - operation: "mtmd_input_chunks_get", - detail: "returned a null chunk within the chunk count", - } - .into() - }) + fn chunks_within_count(&self) -> Result, MtmdEvalError> { + (0..self.len()) + .map(|index| { + self.get(index).ok_or_else(|| { + llama_cpp_ffi_status::FfiContractError { + operation: "mtmd_input_chunks_get", + detail: "returned a null chunk within the chunk count", + } + .into() + }) + }) + .collect() } } diff --git a/llama-cpp-bindings/src/sampling.rs b/llama-cpp-bindings/src/sampling.rs index 5c657fc29..35bbbd660 100644 --- a/llama-cpp-bindings/src/sampling.rs +++ b/llama-cpp-bindings/src/sampling.rs @@ -2252,6 +2252,20 @@ mod sampler_application_tests { ); } + #[test] + fn an_exception_llama_cpp_throws_while_applying_a_sampler_is_reported() { + let greedy_sampler = LlamaSampler::greedy().unwrap(); + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_APPLY_THROWS); + + assert_eq!( + greedy_sampler.apply(&mut excluded_candidates()), + Err(SampleError::SamplerApply(SamplerApplyError::Reported { + message: injected_fault_message(), + })) + ); + } + #[test] fn selecting_samplers_llama_cpp_cannot_build_are_reported() { let distribution_failure = { From ad61507d11fff76e2ad5330eef3953abbbaed38f Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sat, 31 Oct 2026 05:47:59 +0100 Subject: [PATCH 28/82] Test that evaluating mtmd chunks reports a chunk llama.cpp does not return before evaluating any --- .../tests/multimodal_chunk_evaluation.rs | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs index 2861bfbcd..b49fc41bd 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs @@ -160,6 +160,44 @@ fn eval_chunks_reports_an_exception_llama_cpp_throws_while_evaluating_a_chunk( Ok(()) } +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_chunks_reports_a_chunk_llama_cpp_does_not_return_before_evaluating_any( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Describe: <__media__>"); + let mut llama_ctx = fixture.build_context()?; + let n_batch = i32::try_from(llama_ctx.n_batch())?; + let evaluation = { + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNKS_GET_RETURNS_NULL, + ); + + chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, true) + }; + + assert_eq!( + evaluation, + Err(MtmdEvalError::FfiContract(FfiContractError { + operation: "mtmd_input_chunks_get", + detail: "returned a null chunk within the chunk count", + })) + ); + assert_eq!(llama_ctx.memory_seq_pos_max(0)?, -1); + + Ok(()) +} + #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, From 0412a0ec120c407ee8f52d9e05bf62f3e8e816b4 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sat, 31 Oct 2026 18:44:07 +0100 Subject: [PATCH 29/82] Drop the remaining crate-root re-exports and pub(crate) items, import macros at module scope, and name tests after what they verify --- Cargo.lock | 2 + llama-cpp-bindings-tests/Cargo.toml | 2 + llama-cpp-bindings-tests/src/chat_answer.rs | 2 +- .../src/chat_answer_generation.rs | 2 +- .../src/context_after_decoding_text.rs | 2 +- .../src/paris_weather_tool_call.rs | 6 +- llama-cpp-bindings-tests/src/prime_memory.rs | 2 +- .../src/prime_memory_with.rs | 2 +- .../src/recognized_answer.rs | 2 +- .../tokenize_prompt_with_synthetic_image.rs | 2 +- .../approximate_tok_env_eos_selection.rs | 4 +- .../tests/backend_initialization.rs | 4 +- .../tests/batch_context_limits.rs | 4 +- .../tests/chat_answer_parsing.rs | 8 +- .../tests/chat_prompt_rendering.rs | 4 +- .../tests/chat_session_input_validation.rs | 4 +- .../tests/chat_session_llama_cpp_failures.rs | 10 +- .../tests/chat_template_lookup.rs | 4 +- .../tests/chat_template_tool_call_support.rs | 4 +- .../tests/completion_usage_accounting.rs | 4 +- .../tests/context_creation_limits.rs | 8 +- .../tests/context_dimensions.rs | 4 +- .../tests/context_synchronization.rs | 4 +- .../tests/debug_formatting.rs | 4 +- .../tests/decode_abort_control.rs | 4 +- .../tests/decode_batch_rejection.rs | 8 +- .../tests/detokenization.rs | 4 +- llama-cpp-bindings-tests/tests/dry_sampler.rs | 4 +- .../tests/duck_typed_tool_calls.rs | 4 +- .../tests/embedding_availability.rs | 4 +- .../tests/embedding_generation.rs | 4 +- .../tests/embedding_retrieval_errors.rs | 4 +- .../tests/embedding_row_width.rs | 4 +- .../tests/encode_batch_rejection.rs | 8 +- .../tests/encoder_requirement.rs | 4 +- .../end_of_generation_token_detection.rs | 4 +- .../tests/generation_end_classification.rs | 4 +- .../tests/grammar_sampler.rs | 6 +- .../tests/hidden_state_extraction.rs | 4 +- .../tests/image_question_answering.rs | 4 +- .../json_schema_constrained_generation.rs | 4 +- .../tests/lazy_grammar_sampler.rs | 4 +- .../tests/llguidance_sampler.rs | 6 +- .../tests/logit_bias_sampler.rs | 4 +- .../tests/logits_availability.rs | 4 +- .../tests/logits_retrieval.rs | 6 +- .../tests/lora_adapters.rs | 6 +- llama-cpp-bindings-tests/tests/main.rs | 4 +- .../tests/memory_availability.rs | 4 +- .../tests/memory_occupancy.rs | 4 +- .../tests/memory_position_bounds.rs | 4 +- ...emory_position_shift_rope_compatibility.rs | 4 +- .../tests/memory_priming.rs | 4 +- .../tests/memory_sequence_edits.rs | 4 +- .../tests/memory_sequence_id_bounds.rs | 4 +- .../tests/model_architecture.rs | 4 +- .../tests/model_derived_data_caching.rs | 4 +- .../tests/model_hyperparameters.rs | 4 +- .../tests/model_loading_errors.rs | 4 +- .../tests/model_metadata.rs | 4 +- .../tests/multimodal_audio.rs | 4 +- .../tests/multimodal_bitmap_construction.rs | 4 +- .../tests/multimodal_chunk_copy.rs | 4 +- .../tests/multimodal_chunk_encoding.rs | 4 +- .../tests/multimodal_chunk_evaluation.rs | 10 +- .../tests/multimodal_chunk_tokenization.rs | 8 +- .../tests/multimodal_fusion.rs | 4 +- .../multimodal_projector_capabilities.rs | 4 +- .../tests/multimodal_projector_loading.rs | 4 +- .../multimodal_prompt_parts_tokenization.rs | 8 +- .../multimodal_reasoning_classification.rs | 4 +- .../tests/multimodal_usage_accounting.rs | 6 +- .../tests/parameter_fitting.rs | 6 +- .../tests/prompt_token_accounting.rs | 6 +- .../tests/reasoning_classification.rs | 4 +- llama-cpp-bindings-tests/tests/reranking.rs | 4 +- .../tests/sampled_token_ingestion.rs | 8 +- .../tests/sampler_token_acceptance.rs | 4 +- .../tests/session_file_path_validation.rs | 4 +- .../tests/session_file_token_capacity.rs | 4 +- .../tests/session_files.rs | 4 +- .../tests/special_tokens.rs | 4 +- .../tests/state_snapshots.rs | 4 +- .../tests/threadpool_detachment.rs | 4 +- .../tests/token_attributes.rs | 4 +- .../tests/token_candidates.rs | 4 +- .../tests/tokenization.rs | 4 +- .../tests/tool_call_classification.rs | 4 +- .../tests/tool_call_grammar.rs | 4 +- .../tests/tool_call_parsing.rs | 10 +- .../tests/unconstrained_token_sampling.rs | 4 +- .../tests/vocabulary_bounds.rs | 4 +- .../tests/vocabulary_enumeration.rs | 4 +- .../tests/vocabulary_type.rs | 4 +- llama-cpp-bindings-types/src/lib.rs | 17 -- llama-cpp-bindings-types/src/token_usage.rs | 2 +- .../src/chat_grammar_trigger.rs | 14 +- .../src/chat_message_parse_outcome.rs | 12 +- llama-cpp-bindings/src/chat_session.rs | 84 +++---- llama-cpp-bindings/src/chat_span_kind.rs | 10 +- .../gemma4_call_block.rs | 8 +- .../glm47_key_value_tags.rs | 6 +- .../known_marker_candidates.rs | 2 +- .../mistral3_arrow_args.rs | 6 +- .../qwen3_json_inside_tool_call.rs | 6 +- .../chat_template_tool_calls/qwen_xml_tags.rs | 6 +- llama-cpp-bindings/src/chat_templates.rs | 21 +- llama-cpp-bindings/src/context.rs | 160 ++++++------- .../src/context/kv_cache_type.rs | 2 +- llama-cpp-bindings/src/context/memory.rs | 147 ++++++------ llama-cpp-bindings/src/context/session.rs | 73 +++--- .../src/context/state_data_error.rs | 7 +- .../src/error/batch_transfer_error.rs | 7 +- .../src/error/chat_session_error.rs | 7 +- .../src/error/chat_templates_error.rs | 7 +- .../src/error/embeddings_error.rs | 7 +- llama-cpp-bindings/src/error/fit_error.rs | 7 +- llama-cpp-bindings/src/error/grammar_error.rs | 7 +- .../src/error/json_schema_to_grammar_error.rs | 7 +- .../src/error/llama_context_load_error.rs | 7 +- .../src/error/llama_model_load_error.rs | 7 +- .../src/error/memory_seq_add_error.rs | 7 +- .../src/error/memory_seq_div_error.rs | 7 +- .../src/error/memory_seq_pos_max_error.rs | 7 +- .../src/error/parse_chat_message_error.rs | 7 +- .../src/error/process_failure.rs | 7 +- llama-cpp-bindings/src/error/sample_error.rs | 7 +- .../src/error/sampler_accept_error.rs | 7 +- .../src/error/sampler_apply_error.rs | 7 +- .../src/error/sampling_error.rs | 7 +- .../src/error/string_to_token_error.rs | 7 +- .../src/json_schema_to_grammar.rs | 53 ++--- .../src/lazy_grammar_triggers.rs | 64 +++-- llama-cpp-bindings/src/llguidance_sampler.rs | 6 +- llama-cpp-bindings/src/model.rs | 64 ++--- .../src/model/llama_chat_message.rs | 2 +- .../src/model/llama_chat_template.rs | 2 +- .../src/model/llama_lora_adapter.rs | 14 +- llama-cpp-bindings/src/model/params.rs | 94 ++++---- .../src/model/params/param_override_value.rs | 57 ++--- llama-cpp-bindings/src/model/split_mode.rs | 55 ++--- llama-cpp-bindings/src/mtmd/eval_chunk.rs | 40 ++-- llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs | 100 ++++---- .../src/mtmd/mtmd_bitmap_error.rs | 7 +- llama-cpp-bindings/src/mtmd/mtmd_context.rs | 143 ++++++------ .../src/mtmd/mtmd_encode_error.rs | 7 +- .../src/mtmd/mtmd_eval_error.rs | 7 +- .../src/mtmd/mtmd_init_error.rs | 7 +- .../src/mtmd/mtmd_input_chunk.rs | 27 ++- .../src/mtmd/mtmd_input_chunk_error.rs | 7 +- .../src/mtmd/mtmd_input_chunks.rs | 27 ++- .../src/mtmd/mtmd_input_chunks_error.rs | 7 +- .../src/mtmd/mtmd_memory_usage.rs | 26 ++- .../src/mtmd/mtmd_memory_usage_error.rs | 7 +- .../src/mtmd/mtmd_tokenize_error.rs | 7 +- llama-cpp-bindings/src/parsed_chat.rs | 16 +- .../src/sampled_token_classifier.rs | 4 +- llama-cpp-bindings/src/sampling.rs | 221 +++++++++--------- llama-cpp-bindings/src/send_logs_to_log.rs | 11 +- llama-cpp-bindings/src/token/data_array.rs | 12 +- llama-cpp-bindings/src/tool_call_format.rs | 24 +- .../src/tool_call_format/bracketed_args.rs | 20 +- .../src/tool_call_format/json_object.rs | 14 +- .../tool_call_format/key_value_xml_tags.rs | 20 +- .../src/tool_call_format/paired_quote_args.rs | 26 +-- .../src/tool_call_format/parse_step.rs | 2 +- .../tool_call_format_outcome.rs | 2 +- .../src/tool_call_format/xml_function_tags.rs | 14 +- llama-cpp-error-recorder/src/frame_stack.rs | 8 +- llama-cpp-error-recorder/src/lib.rs | 4 - llama-cpp-fault-injection/src/lib.rs | 3 - llama-cpp-ffi-status/src/lib.rs | 4 - .../src/read_and_free_cpp_string.rs | 2 +- llama-cpp-gbnf/src/gbnf_validation_error.rs | 7 +- llama-cpp-gbnf/src/validate_gbnf.rs | 10 +- llama-cpp-log-decoder/src/log_decoder.rs | 2 +- llama-cpp-test-harness-macros/src/expand.rs | 50 ++-- .../src/parsed_load_mode.rs | 28 ++- llama-cpp-test-harness/Cargo.toml | 2 +- llama-cpp-test-harness/src/execution_phase.rs | 5 +- llama-cpp-test-harness/src/execution_plan.rs | 3 +- llama-cpp-test-harness/src/lib.rs | 23 -- .../src/llama_test_registration.rs | 4 +- .../src/llama_tests_main_macro.rs | 2 +- llama-cpp-test-harness/src/run.rs | 2 +- ...ness_self_test.rs => phase_conclusions.rs} | 14 +- llama-cpp-wrapper-error-fixture/src/lib.rs | 2 - 187 files changed, 1291 insertions(+), 1236 deletions(-) rename llama-cpp-test-harness/tests/{harness_self_test.rs => phase_conclusions.rs} (92%) diff --git a/Cargo.lock b/Cargo.lock index 7373be103..9a01275ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1213,6 +1213,7 @@ version = "0.26.0" dependencies = [ "anyhow", "encoding_rs", + "inventory", "llama-cpp-bindings", "llama-cpp-bindings-sys", "llama-cpp-bindings-types", @@ -1220,6 +1221,7 @@ dependencies = [ "llama-cpp-ffi-status", "llama-cpp-gbnf", "llama-cpp-test-harness", + "llama-cpp-test-harness-macros", "serde_json", "toktrie", ] diff --git a/llama-cpp-bindings-tests/Cargo.toml b/llama-cpp-bindings-tests/Cargo.toml index ded310e2e..53c5d036f 100644 --- a/llama-cpp-bindings-tests/Cargo.toml +++ b/llama-cpp-bindings-tests/Cargo.toml @@ -10,6 +10,7 @@ autotests = false [dependencies] anyhow = { workspace = true } encoding_rs = { workspace = true } +inventory = { workspace = true } llama-cpp-bindings = { workspace = true } llama-cpp-bindings-sys = { workspace = true } llama-cpp-bindings-types = { workspace = true } @@ -17,6 +18,7 @@ llama-cpp-fault-injection = { workspace = true } llama-cpp-ffi-status = { workspace = true } llama-cpp-gbnf = { workspace = true } llama-cpp-test-harness = { workspace = true } +llama-cpp-test-harness-macros = { workspace = true } serde_json = { workspace = true } toktrie = { workspace = true } diff --git a/llama-cpp-bindings-tests/src/chat_answer.rs b/llama-cpp-bindings-tests/src/chat_answer.rs index 94b1677b4..5b2f9ce3a 100644 --- a/llama-cpp-bindings-tests/src/chat_answer.rs +++ b/llama-cpp-bindings-tests/src/chat_answer.rs @@ -1,6 +1,6 @@ use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; use llama_cpp_bindings::chat_session::ChatSession; -use llama_cpp_bindings_types::TokenUsage; +use llama_cpp_bindings_types::token_usage::TokenUsage; use crate::classify_sample_loop_outcome::ClassifySampleLoopOutcome; use crate::recognized_answer::recognized_answer; diff --git a/llama-cpp-bindings-tests/src/chat_answer_generation.rs b/llama-cpp-bindings-tests/src/chat_answer_generation.rs index e1cbe795d..6055dff07 100644 --- a/llama-cpp-bindings-tests/src/chat_answer_generation.rs +++ b/llama-cpp-bindings-tests/src/chat_answer_generation.rs @@ -4,7 +4,7 @@ use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; -use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; use crate::chat_answer::ChatAnswer; use crate::classify_sample_loop::ClassifySampleLoop; diff --git a/llama-cpp-bindings-tests/src/context_after_decoding_text.rs b/llama-cpp-bindings-tests/src/context_after_decoding_text.rs index 7653c912c..3e8d5d086 100644 --- a/llama-cpp-bindings-tests/src/context_after_decoding_text.rs +++ b/llama-cpp-bindings-tests/src/context_after_decoding_text.rs @@ -4,7 +4,7 @@ use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; -use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; const TEXT: &str = "The parcel arrived on the last working day of the week."; diff --git a/llama-cpp-bindings-tests/src/paris_weather_tool_call.rs b/llama-cpp-bindings-tests/src/paris_weather_tool_call.rs index 9b1b4e919..a4b7477d9 100644 --- a/llama-cpp-bindings-tests/src/paris_weather_tool_call.rs +++ b/llama-cpp-bindings-tests/src/paris_weather_tool_call.rs @@ -1,8 +1,8 @@ use serde_json::json; -use llama_cpp_bindings_types::ParsedToolCall; -use llama_cpp_bindings_types::ParsedToolCallId; -use llama_cpp_bindings_types::ToolCallArguments; +use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; +use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; +use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; /// The `get_weather` call for Paris that the tool-call fixtures of every format encode. #[must_use] diff --git a/llama-cpp-bindings-tests/src/prime_memory.rs b/llama-cpp-bindings-tests/src/prime_memory.rs index 5f1d78ac5..f731145b0 100644 --- a/llama-cpp-bindings-tests/src/prime_memory.rs +++ b/llama-cpp-bindings-tests/src/prime_memory.rs @@ -1,7 +1,7 @@ use anyhow::Result; use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; use crate::prime_memory_with::prime_memory_with; diff --git a/llama-cpp-bindings-tests/src/prime_memory_with.rs b/llama-cpp-bindings-tests/src/prime_memory_with.rs index 32ed538e6..09b47afe5 100644 --- a/llama-cpp-bindings-tests/src/prime_memory_with.rs +++ b/llama-cpp-bindings-tests/src/prime_memory_with.rs @@ -5,7 +5,7 @@ use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; -use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; /// # Errors /// Forwards tokenization, batch assembly, and [`LlamaContext::decode`] errors verbatim. diff --git a/llama-cpp-bindings-tests/src/recognized_answer.rs b/llama-cpp-bindings-tests/src/recognized_answer.rs index 407b89633..36fb57893 100644 --- a/llama-cpp-bindings-tests/src/recognized_answer.rs +++ b/llama-cpp-bindings-tests/src/recognized_answer.rs @@ -1,7 +1,7 @@ use llama_cpp_bindings::chat_output::ChatOutput; use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; use llama_cpp_bindings::chat_session::ChatSession; -use llama_cpp_bindings_types::ParsedChatMessage; +use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; /// Parses `answer` with the session's parser. /// diff --git a/llama-cpp-bindings-tests/src/tokenize_prompt_with_synthetic_image.rs b/llama-cpp-bindings-tests/src/tokenize_prompt_with_synthetic_image.rs index d959eebe5..0c47de71f 100644 --- a/llama-cpp-bindings-tests/src/tokenize_prompt_with_synthetic_image.rs +++ b/llama-cpp-bindings-tests/src/tokenize_prompt_with_synthetic_image.rs @@ -1,7 +1,7 @@ use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; use llama_cpp_bindings::mtmd::mtmd_input_chunks::MtmdInputChunks; use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; -use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; /// Tokenizes `prompt` with a uniformly grey 64x64 image as its media. /// diff --git a/llama-cpp-bindings-tests/tests/approximate_tok_env_eos_selection.rs b/llama-cpp-bindings-tests/tests/approximate_tok_env_eos_selection.rs index b46dfef4c..991afb27d 100644 --- a/llama-cpp-bindings-tests/tests/approximate_tok_env_eos_selection.rs +++ b/llama-cpp-bindings-tests/tests/approximate_tok_env_eos_selection.rs @@ -11,8 +11,8 @@ use llama_cpp_bindings::gguf_context::GgufContext; use llama_cpp_bindings::model::LlamaModel; use llama_cpp_bindings::model::params::LlamaModelParams; use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/backend_initialization.rs b/llama-cpp-bindings-tests/tests/backend_initialization.rs index e1b030876..bf85c0974 100644 --- a/llama-cpp-bindings-tests/tests/backend_initialization.rs +++ b/llama-cpp-bindings-tests/tests/backend_initialization.rs @@ -1,5 +1,5 @@ -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/batch_context_limits.rs b/llama-cpp-bindings-tests/tests/batch_context_limits.rs index 648a6794e..f65fe8054 100644 --- a/llama-cpp-bindings-tests/tests/batch_context_limits.rs +++ b/llama-cpp-bindings-tests/tests/batch_context_limits.rs @@ -6,8 +6,8 @@ use llama_cpp_bindings::error::decode_error::DecodeError; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs b/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs index 78c0bb30e..bbdb82fe8 100644 --- a/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs +++ b/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs @@ -9,10 +9,10 @@ use llama_cpp_bindings::error::parse_chat_message_error::ParseChatMessageError; use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; use llama_cpp_bindings_tests::weather_chat_session::weather_chat_session; -use llama_cpp_bindings_types::ParsedToolCall; -use llama_cpp_fault_injection::ArmedFault; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs b/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs index 6bb3da7a3..8c6f31169 100644 --- a/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs +++ b/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs @@ -6,8 +6,8 @@ use llama_cpp_bindings::chat_session_options::ChatSessionOptions; use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs index 5ef552a13..91aa724a6 100644 --- a/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs +++ b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs @@ -13,8 +13,8 @@ use llama_cpp_bindings::error::string_to_token_error::StringToTokenError; use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; use llama_cpp_bindings_tests::media_question_chat_session::media_question_chat_session; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs b/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs index 503940a04..40bff51ad 100644 --- a/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs +++ b/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs @@ -10,11 +10,11 @@ use llama_cpp_bindings::error::parse_chat_message_error::ParseChatMessageError; use llama_cpp_bindings::error::sampler_accept_error::SamplerAcceptError; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; use llama_cpp_bindings_tests::weather_chat_session::weather_chat_session; -use llama_cpp_fault_injection::ArmedFault; -use llama_cpp_fault_injection::injected_fault_message; -use llama_cpp_ffi_status::FfiStatusError; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; fn not_utf8() -> Utf8Error { String::from_utf8(vec![0xff]) diff --git a/llama-cpp-bindings-tests/tests/chat_template_lookup.rs b/llama-cpp-bindings-tests/tests/chat_template_lookup.rs index a53042961..9db306a3e 100644 --- a/llama-cpp-bindings-tests/tests/chat_template_lookup.rs +++ b/llama-cpp-bindings-tests/tests/chat_template_lookup.rs @@ -1,8 +1,8 @@ use std::ffi::CString; use llama_cpp_bindings::error::chat_template_error::ChatTemplateError; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/chat_template_tool_call_support.rs b/llama-cpp-bindings-tests/tests/chat_template_tool_call_support.rs index 3eea826d9..f0c5e8307 100644 --- a/llama-cpp-bindings-tests/tests/chat_template_tool_call_support.rs +++ b/llama-cpp-bindings-tests/tests/chat_template_tool_call_support.rs @@ -1,7 +1,7 @@ use anyhow::Result; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), diff --git a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs index 1f15a98a5..d2e9fd525 100644 --- a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs +++ b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs @@ -15,8 +15,8 @@ use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/context_creation_limits.rs b/llama-cpp-bindings-tests/tests/context_creation_limits.rs index 3453282bf..72f723a8e 100644 --- a/llama-cpp-bindings-tests/tests/context_creation_limits.rs +++ b/llama-cpp-bindings-tests/tests/context_creation_limits.rs @@ -2,10 +2,10 @@ use std::num::NonZeroU32; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::error::llama_context_load_error::LlamaContextLoadError; -use llama_cpp_fault_injection::ArmedFault; -use llama_cpp_fault_injection::injected_fault_message; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/context_dimensions.rs b/llama-cpp-bindings-tests/tests/context_dimensions.rs index 13b01877e..f2dedfe84 100644 --- a/llama-cpp-bindings-tests/tests/context_dimensions.rs +++ b/llama-cpp-bindings-tests/tests/context_dimensions.rs @@ -1,8 +1,8 @@ use anyhow::Result; use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/context_synchronization.rs b/llama-cpp-bindings-tests/tests/context_synchronization.rs index 38855784e..11c216b72 100644 --- a/llama-cpp-bindings-tests/tests/context_synchronization.rs +++ b/llama-cpp-bindings-tests/tests/context_synchronization.rs @@ -1,8 +1,8 @@ use anyhow::Result; use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/debug_formatting.rs b/llama-cpp-bindings-tests/tests/debug_formatting.rs index 6dd78d890..810590689 100644 --- a/llama-cpp-bindings-tests/tests/debug_formatting.rs +++ b/llama-cpp-bindings-tests/tests/debug_formatting.rs @@ -1,8 +1,8 @@ use anyhow::Result; use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/decode_abort_control.rs b/llama-cpp-bindings-tests/tests/decode_abort_control.rs index 5962a7640..71ba8f1ca 100644 --- a/llama-cpp-bindings-tests/tests/decode_abort_control.rs +++ b/llama-cpp-bindings-tests/tests/decode_abort_control.rs @@ -9,8 +9,8 @@ use llama_cpp_bindings::error::decode_error::DecodeError; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs b/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs index e413fd4fc..3f163878e 100644 --- a/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs +++ b/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs @@ -8,10 +8,10 @@ use llama_cpp_bindings::error::decode_error::DecodeError; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; -use llama_cpp_fault_injection::ArmedFault; -use llama_cpp_fault_injection::injected_fault_message; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/detokenization.rs b/llama-cpp-bindings-tests/tests/detokenization.rs index a6cd0258b..497c7cd36 100644 --- a/llama-cpp-bindings-tests/tests/detokenization.rs +++ b/llama-cpp-bindings-tests/tests/detokenization.rs @@ -8,8 +8,8 @@ use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/dry_sampler.rs b/llama-cpp-bindings-tests/tests/dry_sampler.rs index 59399cdbe..01d4d36cc 100644 --- a/llama-cpp-bindings-tests/tests/dry_sampler.rs +++ b/llama-cpp-bindings-tests/tests/dry_sampler.rs @@ -2,8 +2,8 @@ use std::ffi::CString; use llama_cpp_bindings::error::sampling_error::SamplingError; use llama_cpp_bindings::sampling::LlamaSampler; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs b/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs index 5062abd41..53deb4dc3 100644 --- a/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs +++ b/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs @@ -5,8 +5,8 @@ use llama_cpp_bindings_tests::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; use llama_cpp_bindings_tests::paris_weather_tool_call::paris_weather_tool_call; use llama_cpp_bindings_tests::recognized_answer::recognized_answer; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; fn assert_duck_types_paris_weather_call(fixture: &LlamaFixture<'_>, answer: &str) -> Result<()> { let session = user_turn_chat_session( diff --git a/llama-cpp-bindings-tests/tests/embedding_availability.rs b/llama-cpp-bindings-tests/tests/embedding_availability.rs index b14d43c69..a714d89b9 100644 --- a/llama-cpp-bindings-tests/tests/embedding_availability.rs +++ b/llama-cpp-bindings-tests/tests/embedding_availability.rs @@ -1,8 +1,8 @@ use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; use llama_cpp_bindings::error::embeddings_error::EmbeddingsError; use llama_cpp_bindings_tests::context_after_decoding_text::context_after_decoding_text; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), diff --git a/llama-cpp-bindings-tests/tests/embedding_generation.rs b/llama-cpp-bindings-tests/tests/embedding_generation.rs index 7cb2c9438..f14843fde 100644 --- a/llama-cpp-bindings-tests/tests/embedding_generation.rs +++ b/llama-cpp-bindings-tests/tests/embedding_generation.rs @@ -9,8 +9,8 @@ use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings_tests::normalize::normalize; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), diff --git a/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs b/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs index 80778ee17..d1e4f2a4e 100644 --- a/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs +++ b/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs @@ -7,8 +7,8 @@ use llama_cpp_bindings::error::embeddings_error::EmbeddingsError; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/embedding_row_width.rs b/llama-cpp-bindings-tests/tests/embedding_row_width.rs index d908d9f90..13ba21988 100644 --- a/llama-cpp-bindings-tests/tests/embedding_row_width.rs +++ b/llama-cpp-bindings-tests/tests/embedding_row_width.rs @@ -2,8 +2,8 @@ use anyhow::Result; use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; use llama_cpp_bindings_tests::context_after_decoding_text::context_after_decoding_text; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; const LFM2_COLBERT_OUTPUT_PROJECTION_WIDTH: usize = 128; const QWEN3_RERANKER_CLASSIFIER_LABELS: usize = 2; diff --git a/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs b/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs index d9d91c4c0..9feb62253 100644 --- a/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs +++ b/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs @@ -6,10 +6,10 @@ use llama_cpp_bindings::error::encode_error::EncodeError; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; -use llama_cpp_fault_injection::ArmedFault; -use llama_cpp_fault_injection::injected_fault_message; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), diff --git a/llama-cpp-bindings-tests/tests/encoder_requirement.rs b/llama-cpp-bindings-tests/tests/encoder_requirement.rs index 4d657a6e8..5f5f99129 100644 --- a/llama-cpp-bindings-tests/tests/encoder_requirement.rs +++ b/llama-cpp-bindings-tests/tests/encoder_requirement.rs @@ -6,8 +6,8 @@ use llama_cpp_bindings::error::encode_error::EncodeError; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs b/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs index bb183fd55..633a4fb20 100644 --- a/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs +++ b/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs @@ -2,8 +2,8 @@ use anyhow::Context; use anyhow::Result; use llama_cpp_bindings::sampled_token::SampledToken; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/generation_end_classification.rs b/llama-cpp-bindings-tests/tests/generation_end_classification.rs index bfd336164..99a84625f 100644 --- a/llama-cpp-bindings-tests/tests/generation_end_classification.rs +++ b/llama-cpp-bindings-tests/tests/generation_end_classification.rs @@ -6,8 +6,8 @@ use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings_tests::ingested_generation::ingested_generation; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; const END_OF_GENERATION_TOKENS: u64 = 1; diff --git a/llama-cpp-bindings-tests/tests/grammar_sampler.rs b/llama-cpp-bindings-tests/tests/grammar_sampler.rs index f9ad0db56..7cb21fc6d 100644 --- a/llama-cpp-bindings-tests/tests/grammar_sampler.rs +++ b/llama-cpp-bindings-tests/tests/grammar_sampler.rs @@ -16,10 +16,10 @@ use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; -use llama_cpp_fault_injection::ArmedFault; +use llama_cpp_fault_injection::armed_fault::ArmedFault; use llama_cpp_gbnf::gbnf_validation_error::GbnfValidationError; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs b/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs index cb3e9e517..a9ee2c57d 100644 --- a/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs +++ b/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs @@ -9,8 +9,8 @@ use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; const STATE: &str = "Shoes arrived two weeks late and in the wrong size."; const FIRST_BRANCH: &str = " Which team should handle this ticket?"; diff --git a/llama-cpp-bindings-tests/tests/image_question_answering.rs b/llama-cpp-bindings-tests/tests/image_question_answering.rs index a0af4001d..756f5ef6c 100644 --- a/llama-cpp-bindings-tests/tests/image_question_answering.rs +++ b/llama-cpp-bindings-tests/tests/image_question_answering.rs @@ -15,8 +15,8 @@ use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; use llama_cpp_bindings_tests::media_question_chat_session::media_question_chat_session; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs index abca8fe3e..fceee06ab 100644 --- a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs +++ b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs @@ -10,8 +10,8 @@ use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs b/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs index 6ef59dd7d..40092bd0d 100644 --- a/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs +++ b/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs @@ -6,8 +6,8 @@ use llama_cpp_bindings::error::grammar_error::GrammarError; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampling::LlamaSampler; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs index 1e71ffaf9..126fddb82 100644 --- a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs +++ b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs @@ -18,9 +18,9 @@ use llama_cpp_bindings::model::params::LlamaModelParams; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_fault_injection::ArmedFault; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; const JSON_SCHEMA: &str = r#"{"type":"object","properties":{"answer":{"type":"string"}},"required":["answer"]}"#; diff --git a/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs b/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs index 68305a8b1..effd70c7e 100644 --- a/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs +++ b/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs @@ -1,6 +1,6 @@ use llama_cpp_bindings::sampling::LlamaSampler; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/logits_availability.rs b/llama-cpp-bindings-tests/tests/logits_availability.rs index a94122170..74e3fc9be 100644 --- a/llama-cpp-bindings-tests/tests/logits_availability.rs +++ b/llama-cpp-bindings-tests/tests/logits_availability.rs @@ -7,8 +7,8 @@ use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings::token::data_array::LlamaTokenDataArray; use llama_cpp_bindings_tests::context_after_decoding_text::context_after_decoding_text; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), diff --git a/llama-cpp-bindings-tests/tests/logits_retrieval.rs b/llama-cpp-bindings-tests/tests/logits_retrieval.rs index 672b17cf4..2c54977b1 100644 --- a/llama-cpp-bindings-tests/tests/logits_retrieval.rs +++ b/llama-cpp-bindings-tests/tests/logits_retrieval.rs @@ -6,9 +6,9 @@ use llama_cpp_bindings::error::logits_error::LogitsError; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; -use llama_cpp_fault_injection::ArmedFault; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/lora_adapters.rs b/llama-cpp-bindings-tests/tests/lora_adapters.rs index 34ba6a43f..4208a3048 100644 --- a/llama-cpp-bindings-tests/tests/lora_adapters.rs +++ b/llama-cpp-bindings-tests/tests/lora_adapters.rs @@ -3,10 +3,10 @@ use anyhow::Result; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::error::llama_lora_adapters_error::LlamaLoraAdaptersError; use llama_cpp_bindings::model::lora_adapter_scale::LoraAdapterScale; -use llama_cpp_fault_injection::ArmedFault; -use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_fault_injection::armed_fault::ArmedFault; use llama_cpp_test_harness::download_model::download_model; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace( diff --git a/llama-cpp-bindings-tests/tests/main.rs b/llama-cpp-bindings-tests/tests/main.rs index 7ca043e98..b683012f1 100644 --- a/llama-cpp-bindings-tests/tests/main.rs +++ b/llama-cpp-bindings-tests/tests/main.rs @@ -82,4 +82,6 @@ mod vocabulary_bounds; mod vocabulary_enumeration; mod vocabulary_type; -llama_cpp_test_harness::llama_tests_main!(); +use llama_cpp_test_harness::llama_tests_main; + +llama_tests_main!(); diff --git a/llama-cpp-bindings-tests/tests/memory_availability.rs b/llama-cpp-bindings-tests/tests/memory_availability.rs index 975776e31..505f946e5 100644 --- a/llama-cpp-bindings-tests/tests/memory_availability.rs +++ b/llama-cpp-bindings-tests/tests/memory_availability.rs @@ -8,8 +8,8 @@ use llama_cpp_bindings::error::memory_seq_cp_error::MemorySeqCpError; use llama_cpp_bindings::error::memory_seq_div_error::MemorySeqDivError; use llama_cpp_bindings::error::memory_seq_pos_max_error::MemorySeqPosMaxError; use llama_cpp_bindings::error::memory_seq_rm_error::MemorySeqRmError; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/memory_occupancy.rs b/llama-cpp-bindings-tests/tests/memory_occupancy.rs index 02614cd56..b236364fb 100644 --- a/llama-cpp-bindings-tests/tests/memory_occupancy.rs +++ b/llama-cpp-bindings-tests/tests/memory_occupancy.rs @@ -1,8 +1,8 @@ use anyhow::Result; use llama_cpp_bindings_tests::prime_memory::prime_memory; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/memory_position_bounds.rs b/llama-cpp-bindings-tests/tests/memory_position_bounds.rs index d8c27f5cc..0daf9c4c5 100644 --- a/llama-cpp-bindings-tests/tests/memory_position_bounds.rs +++ b/llama-cpp-bindings-tests/tests/memory_position_bounds.rs @@ -7,8 +7,8 @@ use llama_cpp_bindings::error::memory_seq_add_error::MemorySeqAddError; use llama_cpp_bindings::error::memory_seq_cp_error::MemorySeqCpError; use llama_cpp_bindings::error::memory_seq_div_error::MemorySeqDivError; use llama_cpp_bindings::error::memory_seq_rm_error::MemorySeqRmError; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs b/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs index 49a274fa0..dcf2a9456 100644 --- a/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs +++ b/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs @@ -6,8 +6,8 @@ use anyhow::Result; use llama_cpp_bindings::error::memory_seq_add_error::MemorySeqAddError; use llama_cpp_bindings::error::memory_seq_div_error::MemorySeqDivError; use llama_cpp_bindings_tests::prime_memory::prime_memory; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/memory_priming.rs b/llama-cpp-bindings-tests/tests/memory_priming.rs index 7a37c5586..2d786870d 100644 --- a/llama-cpp-bindings-tests/tests/memory_priming.rs +++ b/llama-cpp-bindings-tests/tests/memory_priming.rs @@ -3,8 +3,8 @@ use anyhow::Result; use llama_cpp_bindings::batch_add_error::BatchAddError; use llama_cpp_bindings::error::string_to_token_error::StringToTokenError; use llama_cpp_bindings_tests::prime_memory_with::prime_memory_with; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs b/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs index 3688a25c8..ef3ab6a1d 100644 --- a/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs +++ b/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs @@ -2,8 +2,8 @@ use anyhow::Result; use llama_cpp_bindings::error::memory_seq_rm_error::MemorySeqRmError; use llama_cpp_bindings_tests::prime_memory::prime_memory; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs b/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs index d548ae979..e734e94c1 100644 --- a/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs +++ b/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs @@ -2,8 +2,8 @@ use anyhow::Result; use llama_cpp_bindings::error::memory_seq_pos_max_error::MemorySeqPosMaxError; use llama_cpp_bindings::error::memory_seq_rm_error::MemorySeqRmError; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/model_architecture.rs b/llama-cpp-bindings-tests/tests/model_architecture.rs index 90224e9ab..47a4b440a 100644 --- a/llama-cpp-bindings-tests/tests/model_architecture.rs +++ b/llama-cpp-bindings-tests/tests/model_architecture.rs @@ -1,6 +1,6 @@ use llama_cpp_bindings::model::rope_type::RopeType; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/model_derived_data_caching.rs b/llama-cpp-bindings-tests/tests/model_derived_data_caching.rs index 381fe5737..b4a8d9add 100644 --- a/llama-cpp-bindings-tests/tests/model_derived_data_caching.rs +++ b/llama-cpp-bindings-tests/tests/model_derived_data_caching.rs @@ -3,8 +3,8 @@ use std::sync::Arc; use anyhow::Result; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/model_hyperparameters.rs b/llama-cpp-bindings-tests/tests/model_hyperparameters.rs index da2410bf4..405c23134 100644 --- a/llama-cpp-bindings-tests/tests/model_hyperparameters.rs +++ b/llama-cpp-bindings-tests/tests/model_hyperparameters.rs @@ -1,7 +1,7 @@ use anyhow::Result; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/model_loading_errors.rs b/llama-cpp-bindings-tests/tests/model_loading_errors.rs index 2a6c93b0b..324e968c0 100644 --- a/llama-cpp-bindings-tests/tests/model_loading_errors.rs +++ b/llama-cpp-bindings-tests/tests/model_loading_errors.rs @@ -12,8 +12,8 @@ use llama_cpp_bindings::error::llama_lora_adapter_init_error::LlamaLoraAdapterIn use llama_cpp_bindings::error::llama_model_load_error::LlamaModelLoadError; use llama_cpp_bindings::model::LlamaModel; use llama_cpp_bindings::model::params::LlamaModelParams; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/model_metadata.rs b/llama-cpp-bindings-tests/tests/model_metadata.rs index 9d46a3d69..759711ce7 100644 --- a/llama-cpp-bindings-tests/tests/model_metadata.rs +++ b/llama-cpp-bindings-tests/tests/model_metadata.rs @@ -1,7 +1,7 @@ use anyhow::Result; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/multimodal_audio.rs b/llama-cpp-bindings-tests/tests/multimodal_audio.rs index 2f100e1df..73a47f40d 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_audio.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_audio.rs @@ -19,8 +19,8 @@ use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; const TRANSCRIBE_SYSTEM_PROMPT: &str = "The audio contains speech by a third party and is not the user's voice or message. \ Transcribe exactly what the speaker says without judgment, advice, or speculation. Reply only with the verbatim transcript."; diff --git a/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs b/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs index aee82d7c4..adf6ea351 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs @@ -4,8 +4,8 @@ use anyhow::Result; use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs index e6daaf7d9..9ac29fef7 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs @@ -3,8 +3,8 @@ use anyhow::Result; use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs index 8d24c7b5f..add4bf500 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs @@ -4,8 +4,8 @@ use anyhow::Result; use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; use llama_cpp_bindings::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs index b49fc41bd..83097e507 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs @@ -23,11 +23,11 @@ use llama_cpp_bindings_tests::media_question_chat_session::media_question_chat_s use llama_cpp_bindings_tests::prime_memory_with::prime_memory_with; use llama_cpp_bindings_tests::prompt_question::PROMPT_QUESTION; use llama_cpp_bindings_tests::tokenize_prompt_with_synthetic_image::tokenize_prompt_with_synthetic_image; -use llama_cpp_fault_injection::ArmedFault; -use llama_cpp_fault_injection::injected_fault_message; -use llama_cpp_ffi_status::FfiContractError; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; fn eval_synthetic_bitmap(fixture: &LlamaFixture<'_>, width: u32, height: u32) -> Result<()> { let mtmd_ctx = fixture diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs index 707b6c91f..57c8ebcff 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs @@ -8,10 +8,10 @@ use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; use llama_cpp_bindings::mtmd::mtmd_tokenize_error::MtmdTokenizeError; use llama_cpp_bindings::token::LlamaToken; use llama_cpp_bindings_tests::tokenize_prompt_with_synthetic_image::tokenize_prompt_with_synthetic_image; -use llama_cpp_fault_injection::ArmedFault; -use llama_cpp_fault_injection::injected_fault_message; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs index 19a8d1975..2380088fd 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs @@ -22,8 +22,8 @@ use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; const RECOGNIZED_ANIMALS: [&str; 3] = ["llama", "alpaca", "sheep"]; diff --git a/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs b/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs index 6794d7e15..4ddbc5411 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs @@ -6,8 +6,8 @@ use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; use llama_cpp_bindings::mtmd::mtmd_capability::MtmdCapability; use llama_cpp_bindings::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs b/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs index 18eded624..38fecd77b 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs @@ -6,8 +6,8 @@ use llama_cpp_bindings::mtmd::mtmd_context_params::MtmdContextParams; use llama_cpp_bindings::mtmd::mtmd_context_params_error::MtmdContextParamsError; use llama_cpp_bindings::mtmd::mtmd_image_token_limit::MtmdImageTokenLimit; use llama_cpp_bindings::mtmd::mtmd_init_error::MtmdInitError; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs b/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs index 136ce05c1..ff50e79ba 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs @@ -14,10 +14,10 @@ use llama_cpp_bindings::mtmd::mtmd_input_part::MtmdInputPart; use llama_cpp_bindings::mtmd::mtmd_input_text_part::MtmdInputTextPart; use llama_cpp_bindings::mtmd::mtmd_tokenize_error::MtmdTokenizeError; use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_fault_injection::ArmedFault; -use llama_cpp_fault_injection::injected_fault_message; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; const SPECIAL_TOKEN_TEXT: &str = "<|im_start|>"; diff --git a/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs index 775f34a92..c7ce0b540 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs @@ -21,8 +21,8 @@ use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; use llama_cpp_bindings_tests::media_question_chat_session::media_question_chat_session; use llama_cpp_bindings_tests::prompt_question::PROMPT_QUESTION; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs b/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs index 6a2b0237e..bfc1bfbb1 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs @@ -14,9 +14,9 @@ use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; use llama_cpp_bindings_tests::prompt_question::PROMPT_QUESTION; -use llama_cpp_bindings_types::TokenUsage; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_bindings_types::token_usage::TokenUsage; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; fn build_multimodal_chunks_and_eval_into_usage( fixture: &LlamaFixture<'_>, diff --git a/llama-cpp-bindings-tests/tests/parameter_fitting.rs b/llama-cpp-bindings-tests/tests/parameter_fitting.rs index bbd54da42..7a7221e5d 100644 --- a/llama-cpp-bindings-tests/tests/parameter_fitting.rs +++ b/llama-cpp-bindings-tests/tests/parameter_fitting.rs @@ -10,9 +10,9 @@ use llama_cpp_bindings::model::params::LlamaModelParams; use llama_cpp_bindings::model::params::fit_extra_model::FitExtraModel; use llama_cpp_bindings::mtmd::mtmd_context_params::MtmdContextParams; use llama_cpp_bindings::mtmd::mtmd_memory_usage::MtmdMemoryUsage; -use llama_cpp_test_harness::GgufSource; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::gguf_source::GgufSource; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs b/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs index 10a742e0e..e93c7d264 100644 --- a/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs +++ b/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs @@ -6,9 +6,9 @@ use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_bindings_types::TokenUsageError; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_bindings_types::token_usage_error::TokenUsageError; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/reasoning_classification.rs b/llama-cpp-bindings-tests/tests/reasoning_classification.rs index af9ba1eea..9b5918d71 100644 --- a/llama-cpp-bindings-tests/tests/reasoning_classification.rs +++ b/llama-cpp-bindings-tests/tests/reasoning_classification.rs @@ -10,8 +10,8 @@ use llama_cpp_bindings_tests::chat_answer_generation::ChatAnswerGeneration; use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; use llama_cpp_bindings_tests::penalized_nucleus_sampler::penalized_nucleus_sampler; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; const ARITHMETIC_QUESTION: &str = "What is 2 + 2?"; const SINGLE_WORD_INSTRUCTION: &str = "Reply with the single word: four. Do not explain."; diff --git a/llama-cpp-bindings-tests/tests/reranking.rs b/llama-cpp-bindings-tests/tests/reranking.rs index 9b6226ec2..bf2552058 100644 --- a/llama-cpp-bindings-tests/tests/reranking.rs +++ b/llama-cpp-bindings-tests/tests/reranking.rs @@ -9,8 +9,8 @@ use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings_tests::normalize::normalize; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; fn cosine_similarity(vec_a: &[f32], vec_b: &[f32]) -> f32 { vec_a diff --git a/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs b/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs index 8b385a86c..e88d7e633 100644 --- a/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs +++ b/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs @@ -21,10 +21,10 @@ use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings::token::LlamaToken; use llama_cpp_bindings::token_piece::TokenPiece; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; -use llama_cpp_fault_injection::ArmedFault; -use llama_cpp_fault_injection::injected_fault_message; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs b/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs index 2c5d50c11..8fe82791d 100644 --- a/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs +++ b/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs @@ -3,8 +3,8 @@ use anyhow::Result; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/session_file_path_validation.rs b/llama-cpp-bindings-tests/tests/session_file_path_validation.rs index 9d0c4aeac..66aa6b10e 100644 --- a/llama-cpp-bindings-tests/tests/session_file_path_validation.rs +++ b/llama-cpp-bindings-tests/tests/session_file_path_validation.rs @@ -4,8 +4,8 @@ use std::path::Path; use anyhow::Result; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs b/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs index 4f050b0f9..259a22a7d 100644 --- a/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs +++ b/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs @@ -7,8 +7,8 @@ use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/session_files.rs b/llama-cpp-bindings-tests/tests/session_files.rs index 054138afd..d9f1982fe 100644 --- a/llama-cpp-bindings-tests/tests/session_files.rs +++ b/llama-cpp-bindings-tests/tests/session_files.rs @@ -7,8 +7,8 @@ use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/special_tokens.rs b/llama-cpp-bindings-tests/tests/special_tokens.rs index f24a7c64a..f96c15a33 100644 --- a/llama-cpp-bindings-tests/tests/special_tokens.rs +++ b/llama-cpp-bindings-tests/tests/special_tokens.rs @@ -4,8 +4,8 @@ use anyhow::Result; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampled_token::SampledToken; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/state_snapshots.rs b/llama-cpp-bindings-tests/tests/state_snapshots.rs index b4a10982a..9b9cee4f4 100644 --- a/llama-cpp-bindings-tests/tests/state_snapshots.rs +++ b/llama-cpp-bindings-tests/tests/state_snapshots.rs @@ -6,8 +6,8 @@ use llama_cpp_bindings::context::state_data_error::StateDataError; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/threadpool_detachment.rs b/llama-cpp-bindings-tests/tests/threadpool_detachment.rs index 879dc7aa9..ab4c5b7dd 100644 --- a/llama-cpp-bindings-tests/tests/threadpool_detachment.rs +++ b/llama-cpp-bindings-tests/tests/threadpool_detachment.rs @@ -1,8 +1,8 @@ use anyhow::Result; use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/token_attributes.rs b/llama-cpp-bindings-tests/tests/token_attributes.rs index 629303028..db93128ff 100644 --- a/llama-cpp-bindings-tests/tests/token_attributes.rs +++ b/llama-cpp-bindings-tests/tests/token_attributes.rs @@ -1,8 +1,8 @@ use anyhow::Context; use anyhow::Result; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/token_candidates.rs b/llama-cpp-bindings-tests/tests/token_candidates.rs index 854a6afce..c0d846a38 100644 --- a/llama-cpp-bindings-tests/tests/token_candidates.rs +++ b/llama-cpp-bindings-tests/tests/token_candidates.rs @@ -8,8 +8,8 @@ use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::token::LlamaToken; use llama_cpp_bindings::token::data::LlamaTokenData; use llama_cpp_bindings::token::data_array::LlamaTokenDataArray; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/tokenization.rs b/llama-cpp-bindings-tests/tests/tokenization.rs index c0751d4e6..038073dbf 100644 --- a/llama-cpp-bindings-tests/tests/tokenization.rs +++ b/llama-cpp-bindings-tests/tests/tokenization.rs @@ -6,8 +6,8 @@ use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/tool_call_classification.rs b/llama-cpp-bindings-tests/tests/tool_call_classification.rs index 2576e1874..bec28e91d 100644 --- a/llama-cpp-bindings-tests/tests/tool_call_classification.rs +++ b/llama-cpp-bindings-tests/tests/tool_call_classification.rs @@ -9,8 +9,8 @@ use llama_cpp_bindings_tests::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; use llama_cpp_bindings_tests::ingested_generation::ingested_generation; use llama_cpp_bindings_tests::recognized_answer::recognized_answer; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; const REASONING_BEFORE_THE_CALL: &str = "Let me check the weather.\n\n\n"; const QWEN_XML_PARIS_WEATHER_CALL: &str = "\n\ diff --git a/llama-cpp-bindings-tests/tests/tool_call_grammar.rs b/llama-cpp-bindings-tests/tests/tool_call_grammar.rs index 87aa24d01..375f20594 100644 --- a/llama-cpp-bindings-tests/tests/tool_call_grammar.rs +++ b/llama-cpp-bindings-tests/tests/tool_call_grammar.rs @@ -10,8 +10,8 @@ use llama_cpp_bindings_tests::paris_weather_tool_call::paris_weather_tool_call; use llama_cpp_bindings_tests::recognized_answer::recognized_answer; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; use llama_cpp_bindings_tests::weather_chat_session::weather_chat_session; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; const MAX_TOOL_CALL_TOKENS: i32 = 128; diff --git a/llama-cpp-bindings-tests/tests/tool_call_parsing.rs b/llama-cpp-bindings-tests/tests/tool_call_parsing.rs index 9d8982c1f..1c77a591b 100644 --- a/llama-cpp-bindings-tests/tests/tool_call_parsing.rs +++ b/llama-cpp-bindings-tests/tests/tool_call_parsing.rs @@ -11,11 +11,11 @@ use llama_cpp_bindings_tests::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; use llama_cpp_bindings_tests::paris_weather_tool_call::paris_weather_tool_call; use llama_cpp_bindings_tests::recognized_answer::recognized_answer; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; -use llama_cpp_bindings_types::ParsedToolCall; -use llama_cpp_bindings_types::ParsedToolCallId; -use llama_cpp_bindings_types::ToolCallArguments; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; +use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; +use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; const WEATHER_QUESTION: &str = "What is the weather in Paris?"; const PLAIN_CONTENT_REFUSAL: &str = "Sorry, I cannot help with that."; diff --git a/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs b/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs index d81bbdde4..5d11a0b8a 100644 --- a/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs +++ b/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs @@ -9,8 +9,8 @@ use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/vocabulary_bounds.rs b/llama-cpp-bindings-tests/tests/vocabulary_bounds.rs index 49a5dc236..c8c357621 100644 --- a/llama-cpp-bindings-tests/tests/vocabulary_bounds.rs +++ b/llama-cpp-bindings-tests/tests/vocabulary_bounds.rs @@ -2,8 +2,8 @@ use llama_cpp_bindings::error::token_attr_error::TokenAttrError; use llama_cpp_bindings::error::token_outside_vocabulary::TokenOutsideVocabulary; use llama_cpp_bindings::error::token_to_string_error::TokenToStringError; use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs b/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs index 156e2e67f..7009d260a 100644 --- a/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs +++ b/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs @@ -1,7 +1,7 @@ use anyhow::Result; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-tests/tests/vocabulary_type.rs b/llama-cpp-bindings-tests/tests/vocabulary_type.rs index 23ade80fb..540f4566d 100644 --- a/llama-cpp-bindings-tests/tests/vocabulary_type.rs +++ b/llama-cpp-bindings-tests/tests/vocabulary_type.rs @@ -1,8 +1,8 @@ use anyhow::Result; use llama_cpp_bindings::model::vocab_type::VocabType; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-bindings-types/src/lib.rs b/llama-cpp-bindings-types/src/lib.rs index 79205641b..caaf9ac37 100644 --- a/llama-cpp-bindings-types/src/lib.rs +++ b/llama-cpp-bindings-types/src/lib.rs @@ -19,20 +19,3 @@ pub mod tool_call_arguments; pub mod tool_call_markers; pub mod tool_call_value_quote; pub mod xml_tags_shape; - -pub use bracketed_json_shape::BracketedJsonShape; -pub use json_object_shape::JsonObjectShape; -pub use key_value_xml_tags_shape::KeyValueXmlTagsShape; -pub use model_provided_tool_call_id::ModelProvidedToolCallId; -pub use model_provided_tool_call_id_error::ModelProvidedToolCallIdError; -pub use paired_quote_shape::PairedQuoteShape; -pub use parsed_chat_message::ParsedChatMessage; -pub use parsed_tool_call::ParsedToolCall; -pub use parsed_tool_call_id::ParsedToolCallId; -pub use token_usage::TokenUsage; -pub use token_usage_error::TokenUsageError; -pub use tool_call_args_shape::ToolCallArgsShape; -pub use tool_call_arguments::ToolCallArguments; -pub use tool_call_markers::ToolCallMarkers; -pub use tool_call_value_quote::ToolCallValueQuote; -pub use xml_tags_shape::XmlTagsShape; diff --git a/llama-cpp-bindings-types/src/token_usage.rs b/llama-cpp-bindings-types/src/token_usage.rs index 55f9a90e1..5bd59a454 100644 --- a/llama-cpp-bindings-types/src/token_usage.rs +++ b/llama-cpp-bindings-types/src/token_usage.rs @@ -217,7 +217,7 @@ mod tests { } #[test] - fn record_cached_equal_to_prompt_succeeds() { + fn caching_every_prompt_token_is_accepted() { let mut usage = TokenUsage::new(); usage.record_prompt_tokens(5); usage diff --git a/llama-cpp-bindings/src/chat_grammar_trigger.rs b/llama-cpp-bindings/src/chat_grammar_trigger.rs index abf16862b..e3c2a3d78 100644 --- a/llama-cpp-bindings/src/chat_grammar_trigger.rs +++ b/llama-cpp-bindings/src/chat_grammar_trigger.rs @@ -1,3 +1,5 @@ +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use crate::token::LlamaToken; #[derive(Clone, Debug, Eq, PartialEq)] @@ -14,12 +16,12 @@ impl ChatGrammarTrigger { /// /// # Errors /// - /// Returns [`llama_cpp_ffi_status::FfiStatusError`] for a trigger type unknown to this binding. + /// Returns [`llama_cpp_ffi_status::ffi_status_error::FfiStatusError`] for a trigger type unknown to this binding. pub fn from_wrapper( trigger_type: llama_cpp_bindings_sys::llama_rs_chat_grammar_trigger_type, value: String, token: impl FnOnce() -> LlamaToken, - ) -> Result { + ) -> Result { match trigger_type { llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_TOKEN => { Ok(Self::Token(token())) @@ -33,7 +35,7 @@ impl ChatGrammarTrigger { llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL => { Ok(Self::PatternFull(value)) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_chat_session_grammar_trigger_type", code: i64::from(other), }), @@ -43,12 +45,14 @@ impl ChatGrammarTrigger { #[cfg(test)] mod tests { + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use crate::chat_grammar_trigger::ChatGrammarTrigger; use crate::token::LlamaToken; fn trigger_of( trigger_type: llama_cpp_bindings_sys::llama_rs_chat_grammar_trigger_type, - ) -> Result { + ) -> Result { ChatGrammarTrigger::from_wrapper(trigger_type, "".to_owned(), || { LlamaToken::new(151_657) }) @@ -77,7 +81,7 @@ mod tests { fn an_unknown_wrapper_trigger_type_is_a_status_error() { assert_eq!( trigger_of(99), - Err(llama_cpp_ffi_status::FfiStatusError { + Err(FfiStatusError { operation: "llama_rs_chat_session_grammar_trigger_type", code: 99, }) diff --git a/llama-cpp-bindings/src/chat_message_parse_outcome.rs b/llama-cpp-bindings/src/chat_message_parse_outcome.rs index 64762a72f..02a35ed09 100644 --- a/llama-cpp-bindings/src/chat_message_parse_outcome.rs +++ b/llama-cpp-bindings/src/chat_message_parse_outcome.rs @@ -1,5 +1,5 @@ -use llama_cpp_bindings_types::ParsedChatMessage; -use llama_cpp_bindings_types::ParsedToolCall; +use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; +use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; use crate::raw_chat_message::RawChatMessage; @@ -39,10 +39,10 @@ impl ChatMessageParseOutcome { mod tests { use serde_json::json; - use llama_cpp_bindings_types::ParsedChatMessage; - use llama_cpp_bindings_types::ParsedToolCall; - use llama_cpp_bindings_types::ParsedToolCallId; - use llama_cpp_bindings_types::ToolCallArguments; + use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; + use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; + use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; + use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; use crate::chat_message_parse_outcome::ChatMessageParseOutcome; use crate::chat_output_completeness::ChatOutputCompleteness; diff --git a/llama-cpp-bindings/src/chat_session.rs b/llama-cpp-bindings/src/chat_session.rs index 7d746720f..7d0213f1c 100644 --- a/llama-cpp-bindings/src/chat_session.rs +++ b/llama-cpp-bindings/src/chat_session.rs @@ -7,8 +7,10 @@ use std::slice; use serde_json::Value; use serde_json::json; -use llama_cpp_bindings_types::ParsedChatMessage; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use crate::chat_assistant_prefill::ChatAssistantPrefill; use crate::chat_assistant_turn::ChatAssistantTurn; @@ -86,11 +88,8 @@ fn session_text(text: *const c_char, length: usize) -> Result llama_cpp_ffi_status::FfiContractError { - llama_cpp_ffi_status::FfiContractError { operation, detail } +const fn contract_violation(operation: &'static str, detail: &'static str) -> FfiContractError { + FfiContractError { operation, detail } } fn chat_session_create_status_to_result( @@ -169,7 +168,7 @@ fn chat_session_create_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_ERROR_ARG => { Err(contract_violation(OPERATION, "was given a null out_error argument").into()) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: OPERATION, code: i64::from(other), } @@ -233,7 +232,7 @@ fn chat_session_parse_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_ERROR_ARG => { Err(contract_violation(OPERATION, "was given a null out_error argument").into()) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: OPERATION, code: i64::from(other), } @@ -257,7 +256,7 @@ fn read_chat_spans( bytes: wrapper_span.start..wrapper_span.end, }) }) - .collect::, llama_cpp_ffi_status::FfiStatusError>>() + .collect::, FfiStatusError>>() .map(|spans_read| ChatSpans { parsed_length: view.parsed_length, spans: spans_read, @@ -320,7 +319,7 @@ fn chat_session_classify_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_ERROR_ARG => { Err(contract_violation(OPERATION, "was given a null out_error argument").into()) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: OPERATION, code: i64::from(other), } @@ -764,6 +763,9 @@ mod tests { use serde_json::json; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::chat_session_classify_status_to_result; use super::chat_session_create_status_to_result; use super::chat_session_parse_status_to_result; @@ -823,12 +825,10 @@ mod tests { assert_eq!( chat_session_create_status_to_result(status, ptr::null_mut(), ptr::null_mut()) .err(), - Some(ChatSessionError::FfiContract( - llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_chat_session_create", - detail - } - )) + Some(ChatSessionError::FfiContract(FfiContractError { + operation: "llama_rs_chat_session_create", + detail + })) ); } } @@ -889,12 +889,10 @@ mod tests { chat_session_create_status_to_result(status, ptr::null_mut(), ptr::null_mut()).err() }), refusals.map(|_status| { - Some(ChatSessionError::FfiContract( - llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_chat_session_create", - detail: "reported a failure without an error message", - }, - )) + Some(ChatSessionError::FfiContract(FfiContractError { + operation: "llama_rs_chat_session_create", + detail: "reported a failure without an error message", + })) }) ); } @@ -923,12 +921,10 @@ mod tests { fn creating_a_session_preserves_an_unknown_status() { assert_eq!( chat_session_create_status_to_result(255, ptr::null_mut(), ptr::null_mut()).err(), - Some(ChatSessionError::FfiStatus( - llama_cpp_ffi_status::FfiStatusError { - operation: "llama_rs_chat_session_create", - code: 255 - } - )) + Some(ChatSessionError::FfiStatus(FfiStatusError { + operation: "llama_rs_chat_session_create", + code: 255 + })) ); } @@ -968,12 +964,10 @@ mod tests { .err() }; let contract = |detail| { - Some(ParseChatMessageError::FfiContract( - llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_chat_session_parse", - detail, - }, - )) + Some(ParseChatMessageError::FfiContract(FfiContractError { + operation: "llama_rs_chat_session_parse", + detail, + })) }; assert_eq!( @@ -1028,7 +1022,7 @@ mod tests { contract("was given a null byte_tokens argument"), contract("was given a null out_handle argument"), contract("was given a null out_error argument"), - Some(ParseChatMessageError::FfiStatus(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_chat_session_parse", code: 255 })), + Some(ParseChatMessageError::FfiStatus(FfiStatusError { operation: "llama_rs_chat_session_parse", code: 255 })), ] ); } @@ -1060,12 +1054,10 @@ mod tests { .err() }; let contract = |detail| { - Some(ParseChatMessageError::FfiContract( - llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_chat_session_classify", - detail, - }, - )) + Some(ParseChatMessageError::FfiContract(FfiContractError { + operation: "llama_rs_chat_session_classify", + detail, + })) }; assert_eq!( @@ -1120,7 +1112,7 @@ mod tests { contract("was given a null byte_tokens argument"), contract("was given a null out_spans argument"), contract("was given a null out_error argument"), - Some(ParseChatMessageError::FfiStatus(llama_cpp_ffi_status::FfiStatusError { operation: "llama_rs_chat_session_classify", code: 255 })), + Some(ParseChatMessageError::FfiStatus(FfiStatusError { operation: "llama_rs_chat_session_classify", code: 255 })), ] ); } @@ -1215,15 +1207,15 @@ mod tests { .err(), ], [ - Some(ParseChatMessageError::FfiContract(llama_cpp_ffi_status::FfiContractError { + Some(ParseChatMessageError::FfiContract(FfiContractError { operation: "llama_rs_chat_session_parse", detail: "reported a failure without an error message", })), - Some(ParseChatMessageError::FfiContract(llama_cpp_ffi_status::FfiContractError { + Some(ParseChatMessageError::FfiContract(FfiContractError { operation: "llama_rs_chat_session_parse", detail: "reported a failure without an error message", })), - Some(ParseChatMessageError::FfiContract(llama_cpp_ffi_status::FfiContractError { + Some(ParseChatMessageError::FfiContract(FfiContractError { operation: "llama_rs_chat_session_classify", detail: "reported a thrown C++ exception without an error message", })), diff --git a/llama-cpp-bindings/src/chat_span_kind.rs b/llama-cpp-bindings/src/chat_span_kind.rs index 96403275a..1048a5306 100644 --- a/llama-cpp-bindings/src/chat_span_kind.rs +++ b/llama-cpp-bindings/src/chat_span_kind.rs @@ -1,3 +1,5 @@ +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use crate::chat_piece_role::ChatPieceRole; use crate::sampled_token_section::SampledTokenSection; @@ -24,7 +26,7 @@ impl ChatSpanKind { } impl TryFrom for ChatSpanKind { - type Error = llama_cpp_ffi_status::FfiStatusError; + type Error = FfiStatusError; fn try_from( kind: llama_cpp_bindings_sys::llama_rs_chat_span_kind, @@ -39,7 +41,7 @@ impl TryFrom for ChatSpanKind { llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL_MARKER => { Ok(Self::ToolCallMarker) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_chat_spans_view_of", code: i64::from(other), }), @@ -49,6 +51,8 @@ impl TryFrom for ChatSpanKind { #[cfg(test)] mod tests { + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::ChatSpanKind; use crate::chat_piece_role::ChatPieceRole; use crate::sampled_token_section::SampledTokenSection; @@ -107,7 +111,7 @@ mod tests { fn an_unknown_wrapper_span_kind_is_a_status_error() { assert_eq!( ChatSpanKind::try_from(99), - Err(llama_cpp_ffi_status::FfiStatusError { + Err(FfiStatusError { operation: "llama_rs_chat_spans_view_of", code: 99, }) diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/gemma4_call_block.rs b/llama-cpp-bindings/src/chat_template_tool_calls/gemma4_call_block.rs index add745253..39603a26f 100644 --- a/llama-cpp-bindings/src/chat_template_tool_calls/gemma4_call_block.rs +++ b/llama-cpp-bindings/src/chat_template_tool_calls/gemma4_call_block.rs @@ -1,7 +1,7 @@ -use llama_cpp_bindings_types::PairedQuoteShape; -use llama_cpp_bindings_types::ToolCallArgsShape; -use llama_cpp_bindings_types::ToolCallMarkers; -use llama_cpp_bindings_types::ToolCallValueQuote; +use llama_cpp_bindings_types::paired_quote_shape::PairedQuoteShape; +use llama_cpp_bindings_types::tool_call_args_shape::ToolCallArgsShape; +use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; +use llama_cpp_bindings_types::tool_call_value_quote::ToolCallValueQuote; pub struct Gemma4CallBlockFormat; diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/glm47_key_value_tags.rs b/llama-cpp-bindings/src/chat_template_tool_calls/glm47_key_value_tags.rs index 437d64b50..fb2e77e8f 100644 --- a/llama-cpp-bindings/src/chat_template_tool_calls/glm47_key_value_tags.rs +++ b/llama-cpp-bindings/src/chat_template_tool_calls/glm47_key_value_tags.rs @@ -1,6 +1,6 @@ -use llama_cpp_bindings_types::KeyValueXmlTagsShape; -use llama_cpp_bindings_types::ToolCallArgsShape; -use llama_cpp_bindings_types::ToolCallMarkers; +use llama_cpp_bindings_types::key_value_xml_tags_shape::KeyValueXmlTagsShape; +use llama_cpp_bindings_types::tool_call_args_shape::ToolCallArgsShape; +use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; pub struct Glm47KeyValueTagsFormat; diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/known_marker_candidates.rs b/llama-cpp-bindings/src/chat_template_tool_calls/known_marker_candidates.rs index f9272dc99..7aa8aba1b 100644 --- a/llama-cpp-bindings/src/chat_template_tool_calls/known_marker_candidates.rs +++ b/llama-cpp-bindings/src/chat_template_tool_calls/known_marker_candidates.rs @@ -1,4 +1,4 @@ -use llama_cpp_bindings_types::ToolCallMarkers; +use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; use crate::chat_template_tool_calls::gemma4_call_block::Gemma4CallBlockFormat; use crate::chat_template_tool_calls::glm47_key_value_tags::Glm47KeyValueTagsFormat; diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/mistral3_arrow_args.rs b/llama-cpp-bindings/src/chat_template_tool_calls/mistral3_arrow_args.rs index d2d9779be..c333e250d 100644 --- a/llama-cpp-bindings/src/chat_template_tool_calls/mistral3_arrow_args.rs +++ b/llama-cpp-bindings/src/chat_template_tool_calls/mistral3_arrow_args.rs @@ -1,6 +1,6 @@ -use llama_cpp_bindings_types::BracketedJsonShape; -use llama_cpp_bindings_types::ToolCallArgsShape; -use llama_cpp_bindings_types::ToolCallMarkers; +use llama_cpp_bindings_types::bracketed_json_shape::BracketedJsonShape; +use llama_cpp_bindings_types::tool_call_args_shape::ToolCallArgsShape; +use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; pub struct Mistral3ArrowArgsFormat; diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/qwen3_json_inside_tool_call.rs b/llama-cpp-bindings/src/chat_template_tool_calls/qwen3_json_inside_tool_call.rs index 69ebc0c83..1aae1f58b 100644 --- a/llama-cpp-bindings/src/chat_template_tool_calls/qwen3_json_inside_tool_call.rs +++ b/llama-cpp-bindings/src/chat_template_tool_calls/qwen3_json_inside_tool_call.rs @@ -1,6 +1,6 @@ -use llama_cpp_bindings_types::JsonObjectShape; -use llama_cpp_bindings_types::ToolCallArgsShape; -use llama_cpp_bindings_types::ToolCallMarkers; +use llama_cpp_bindings_types::json_object_shape::JsonObjectShape; +use llama_cpp_bindings_types::tool_call_args_shape::ToolCallArgsShape; +use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; pub struct Qwen3JsonInsideToolCallFormat; diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/qwen_xml_tags.rs b/llama-cpp-bindings/src/chat_template_tool_calls/qwen_xml_tags.rs index 7211a221b..5db3cd41e 100644 --- a/llama-cpp-bindings/src/chat_template_tool_calls/qwen_xml_tags.rs +++ b/llama-cpp-bindings/src/chat_template_tool_calls/qwen_xml_tags.rs @@ -1,6 +1,6 @@ -use llama_cpp_bindings_types::ToolCallArgsShape; -use llama_cpp_bindings_types::ToolCallMarkers; -use llama_cpp_bindings_types::XmlTagsShape; +use llama_cpp_bindings_types::tool_call_args_shape::ToolCallArgsShape; +use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; +use llama_cpp_bindings_types::xml_tags_shape::XmlTagsShape; pub struct QwenXmlTagsFormat; diff --git a/llama-cpp-bindings/src/chat_templates.rs b/llama-cpp-bindings/src/chat_templates.rs index 156100324..70830b31c 100644 --- a/llama-cpp-bindings/src/chat_templates.rs +++ b/llama-cpp-bindings/src/chat_templates.rs @@ -2,7 +2,9 @@ use std::ffi::c_char; use std::ptr; use std::ptr::NonNull; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use crate::error::chat_templates_error::ChatTemplatesError; use crate::model::LlamaModel; @@ -15,7 +17,7 @@ fn chat_templates_init_status_to_result( match status { llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_OK => NonNull::new(out_templates) .ok_or_else(|| { - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_chat_templates_init", detail: "success status contained a null templates handle", } @@ -41,27 +43,27 @@ fn chat_templates_init_status_to_result( Err(ChatTemplatesError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_MODEL_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_chat_templates_init", detail: "was given a null model argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_TEMPLATES_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_chat_templates_init", detail: "was given a null out_templates argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_ERROR_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_chat_templates_init", detail: "was given a null out_error argument", } .into()) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_chat_templates_init", code: i64::from(other), } @@ -128,6 +130,9 @@ mod tests { use std::ffi::c_char; use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::chat_templates_init_status_to_result; use crate::error::chat_templates_error::ChatTemplatesError; @@ -139,7 +144,7 @@ mod tests { } const fn contract(detail: &'static str) -> ChatTemplatesError { - ChatTemplatesError::FfiContract(llama_cpp_ffi_status::FfiContractError { + ChatTemplatesError::FfiContract(FfiContractError { operation: "llama_rs_chat_templates_init", detail, }) @@ -194,7 +199,7 @@ mod tests { Some(contract("was given a null model argument")), Some(contract("was given a null out_templates argument")), Some(contract("was given a null out_error argument")), - Some(ChatTemplatesError::FfiStatus(llama_cpp_ffi_status::FfiStatusError { + Some(ChatTemplatesError::FfiStatus(FfiStatusError { operation: "llama_rs_chat_templates_init", code: 255, })), diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index 9a47bb51c..e2bb287c5 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -29,7 +29,11 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use log::debug; + +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use crate::context::context_limits::ContextLimits; use crate::context::decoded_output_rows::DecodedOutputRows; @@ -71,21 +75,21 @@ fn nextn_hidden_state_probe_status_to_result( Ok(produces_nextn_hidden_state) } llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_CTX_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_context_graph_produces_nextn_hidden_state", detail: "was given a null ctx argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_PRODUCES_NEXTN_HIDDEN_STATE_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_context_graph_produces_nextn_hidden_state", detail: "was given a null out_produces_nextn_hidden_state argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_ERROR_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_context_graph_produces_nextn_hidden_state", detail: "was given a null out_error argument", } @@ -110,7 +114,7 @@ fn nextn_hidden_state_probe_status_to_result( }?; Err(EmbeddingsError::NextnProbeReported { message }) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_context_graph_produces_nextn_hidden_state", code: i64::from(other), } @@ -126,7 +130,7 @@ fn new_context_with_model_status_to_result( match status { llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_OK => NonNull::new(out_ctx) .ok_or_else(|| { - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_new_context_with_model", detail: "success status contained a null context", } @@ -152,27 +156,27 @@ fn new_context_with_model_status_to_result( Err(LlamaContextLoadError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_NULL_MODEL_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_new_context_with_model", detail: "was given a null model argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_NULL_OUT_CTX_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_new_context_with_model", detail: "was given a null out_ctx argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_NULL_OUT_ERROR_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_new_context_with_model", detail: "was given a null out_error argument", } .into()) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_new_context_with_model", code: i64::from(other), } @@ -188,7 +192,7 @@ fn batch_ext_init_status_to_result( match status { llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_OK => { NonNull::new(out_batch).ok_or_else(|| { - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_batch_ext_init", detail: "success status contained a null batch", } @@ -211,28 +215,26 @@ fn batch_ext_init_status_to_result( }?; Err(LlamaContextLoadError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_NULL_CTX_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_batch_ext_init", - detail: "was given a null ctx argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_NULL_CTX_ARG => Err(FfiContractError { + operation: "llama_rs_batch_ext_init", + detail: "was given a null ctx argument", } + .into()), llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_NULL_OUT_BATCH_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_batch_ext_init", detail: "was given a null out_batch argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_NULL_OUT_ERROR_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_batch_ext_init", detail: "was given a null out_error argument", } .into()) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_batch_ext_init", code: i64::from(other), } @@ -245,7 +247,7 @@ fn batch_ext_assign_entries_status_to_result( out_error: *mut c_char, ) -> Result<(), BatchTransferError> { let refused_validated_entry = |detail: &'static str| -> Result<(), BatchTransferError> { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_batch_ext_assign_entries", detail, } @@ -311,7 +313,7 @@ fn batch_ext_assign_entries_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_ERROR_ARG => { refused_validated_entry("was given a null out_error argument") } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_batch_ext_assign_entries", code: i64::from(other), } @@ -330,7 +332,7 @@ where const OPERATION: &str = "llama_rs_process"; let contract_violation = |detail: &'static str| { - ProcessFailure::from(llama_cpp_ffi_status::FfiContractError { + ProcessFailure::from(FfiContractError { operation: OPERATION, detail, }) @@ -375,7 +377,7 @@ where llama_cpp_bindings_sys::LLAMA_RS_PROCESS_NULL_OUT_ERROR_ARG => { Err(contract_violation("was given a null out_error argument").into()) } - other => Err(ProcessFailure::from(llama_cpp_ffi_status::FfiStatusError { + other => Err(ProcessFailure::from(FfiStatusError { operation: OPERATION, code: i64::from(other), }) @@ -955,7 +957,7 @@ impl<'model> LlamaContext<'model> { }; check_lora_adapters_result(err_code)?; - log::debug!("Updated lora adapters"); + debug!("Updated lora adapters"); Ok(()) } } @@ -975,12 +977,12 @@ mod check_lora_adapters_result_tests { use crate::error::llama_lora_adapters_error::LlamaLoraAdaptersError; #[test] - fn check_lora_adapters_result_ok_for_zero() { + fn a_zero_set_adapters_return_code_is_success() { assert!(check_lora_adapters_result(0).is_ok()); } #[test] - fn check_lora_adapters_result_error_for_nonzero() { + fn a_nonzero_set_adapters_return_code_is_an_error() { let result = check_lora_adapters_result(-1); assert_eq!(result, Err(LlamaLoraAdaptersError::ErrorResult(-1))); @@ -991,8 +993,10 @@ mod check_lora_adapters_result_tests { mod nextn_hidden_state_probe_status_to_result_tests { use std::ptr; - use llama_cpp_ffi_status::read_and_free_cpp_string; - use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; + use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer::wrapper_allocated_error_message_pointer; use super::nextn_hidden_state_probe_status_to_result; use crate::error::embeddings_error::EmbeddingsError; @@ -1017,7 +1021,7 @@ mod nextn_hidden_state_probe_status_to_result_tests { false, ptr::null_mut(), ), - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_context_graph_produces_nextn_hidden_state", detail: "was given a null ctx argument", } @@ -1033,7 +1037,7 @@ mod nextn_hidden_state_probe_status_to_result_tests { false, ptr::null_mut(), ), - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_context_graph_produces_nextn_hidden_state", detail: "was given a null out_produces_nextn_hidden_state argument", } @@ -1049,7 +1053,7 @@ mod nextn_hidden_state_probe_status_to_result_tests { false, ptr::null_mut(), ), - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_context_graph_produces_nextn_hidden_state", detail: "was given a null out_error argument", } @@ -1125,7 +1129,7 @@ mod nextn_hidden_state_probe_status_to_result_tests { false, ptr::null_mut(), ), - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_context_graph_produces_nextn_hidden_state", detail: "reported a thrown C++ exception without an error message", } @@ -1137,7 +1141,7 @@ mod nextn_hidden_state_probe_status_to_result_tests { fn nextn_hidden_state_probe_status_to_result_maps_an_unknown_status() { assert_eq!( nextn_hidden_state_probe_status_to_result(255, false, ptr::null_mut()), - Err(llama_cpp_ffi_status::FfiStatusError { + Err(FfiStatusError { operation: "llama_rs_context_graph_produces_nextn_hidden_state", code: 255, } @@ -1150,6 +1154,9 @@ mod nextn_hidden_state_probe_status_to_result_tests { mod new_context_with_model_status_to_result_tests { use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::new_context_with_model_status_to_result; use crate::error::llama_context_load_error::LlamaContextLoadError; @@ -1163,12 +1170,10 @@ mod new_context_with_model_status_to_result_tests { assert_eq!( result, - Err(LlamaContextLoadError::FfiContract( - llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_new_context_with_model", - detail: "success status contained a null context", - } - )) + Err(LlamaContextLoadError::FfiContract(FfiContractError { + operation: "llama_rs_new_context_with_model", + detail: "success status contained a null context", + })) ); } @@ -1223,7 +1228,7 @@ mod new_context_with_model_status_to_result_tests { assert_eq!( result, - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_new_context_with_model", detail: "reported a thrown C++ exception without an error message", } @@ -1237,12 +1242,10 @@ mod new_context_with_model_status_to_result_tests { assert_eq!( result, - Err(LlamaContextLoadError::FfiStatus( - llama_cpp_ffi_status::FfiStatusError { - operation: "llama_rs_new_context_with_model", - code: 255, - } - )) + Err(LlamaContextLoadError::FfiStatus(FfiStatusError { + operation: "llama_rs_new_context_with_model", + code: 255, + })) ); } @@ -1256,7 +1259,7 @@ mod new_context_with_model_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_new_context_with_model", detail: "was given a null model argument", } @@ -1275,7 +1278,7 @@ mod new_context_with_model_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_new_context_with_model", detail: "was given a null out_ctx argument", } @@ -1294,7 +1297,7 @@ mod new_context_with_model_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_new_context_with_model", detail: "was given a null out_error argument", } @@ -1321,6 +1324,9 @@ mod new_context_with_model_status_to_result_tests { mod batch_ext_init_status_to_result_tests { use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::batch_ext_init_status_to_result; use crate::error::llama_context_load_error::LlamaContextLoadError; @@ -1346,12 +1352,10 @@ mod batch_ext_init_status_to_result_tests { ] { assert_eq!( batch_ext_init_status_to_result(status, ptr::null_mut(), ptr::null_mut()).err(), - Some(LlamaContextLoadError::FfiContract( - llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_batch_ext_init", - detail, - } - )) + Some(LlamaContextLoadError::FfiContract(FfiContractError { + operation: "llama_rs_batch_ext_init", + detail, + })) ); } } @@ -1404,12 +1408,10 @@ mod batch_ext_init_status_to_result_tests { ptr::null_mut(), ) .err(), - Some(LlamaContextLoadError::FfiContract( - llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_batch_ext_init", - detail: "reported a thrown C++ exception without an error message", - } - )) + Some(LlamaContextLoadError::FfiContract(FfiContractError { + operation: "llama_rs_batch_ext_init", + detail: "reported a thrown C++ exception without an error message", + })) ); } @@ -1417,12 +1419,10 @@ mod batch_ext_init_status_to_result_tests { fn initializing_a_batch_preserves_an_unknown_status() { assert_eq!( batch_ext_init_status_to_result(255, ptr::null_mut(), ptr::null_mut()).err(), - Some(LlamaContextLoadError::FfiStatus( - llama_cpp_ffi_status::FfiStatusError { - operation: "llama_rs_batch_ext_init", - code: 255, - } - )) + Some(LlamaContextLoadError::FfiStatus(FfiStatusError { + operation: "llama_rs_batch_ext_init", + code: 255, + })) ); } } @@ -1431,6 +1431,9 @@ mod batch_ext_init_status_to_result_tests { mod batch_ext_assign_entries_status_to_result_tests { use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::batch_ext_assign_entries_status_to_result; use crate::error::batch_transfer_error::BatchTransferError; @@ -1492,7 +1495,7 @@ mod batch_ext_assign_entries_status_to_result_tests { ] { assert_eq!( batch_ext_assign_entries_status_to_result(status, ptr::null_mut()), - Err(BatchTransferError::FfiContract(llama_cpp_ffi_status::FfiContractError { + Err(BatchTransferError::FfiContract(FfiContractError { operation: "llama_rs_batch_ext_assign_entries", detail, })) @@ -1540,7 +1543,7 @@ mod batch_ext_assign_entries_status_to_result_tests { ptr::null_mut(), ), Err(BatchTransferError::FfiContract( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_batch_ext_assign_entries", detail: "reported a thrown C++ exception without an error message", } @@ -1552,12 +1555,10 @@ mod batch_ext_assign_entries_status_to_result_tests { fn adding_a_batch_entry_preserves_an_unknown_status() { assert_eq!( batch_ext_assign_entries_status_to_result(255, ptr::null_mut()), - Err(BatchTransferError::FfiStatus( - llama_cpp_ffi_status::FfiStatusError { - operation: "llama_rs_batch_ext_assign_entries", - code: 255, - } - )) + Err(BatchTransferError::FfiStatus(FfiStatusError { + operation: "llama_rs_batch_ext_assign_entries", + code: 255, + })) ); } } @@ -1566,6 +1567,9 @@ mod batch_ext_assign_entries_status_to_result_tests { mod process_status_to_result_tests { use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::process_status_to_result; use crate::error::decode_error::DecodeError; use crate::error::encode_error::EncodeError; @@ -1574,7 +1578,7 @@ mod process_status_to_result_tests { const OPERATION: &str = "llama_rs_process"; fn contract_violation(detail: &'static str) -> DecodeError { - ProcessFailure::from(llama_cpp_ffi_status::FfiContractError { + ProcessFailure::from(FfiContractError { operation: OPERATION, detail, }) @@ -1618,7 +1622,7 @@ mod process_status_to_result_tests { "was given a null out_llama_cpp_return_code argument" )), Err(contract_violation("was given a null out_error argument")), - Err(ProcessFailure::from(llama_cpp_ffi_status::FfiStatusError { + Err(ProcessFailure::from(FfiStatusError { operation: OPERATION, code: 255, }) diff --git a/llama-cpp-bindings/src/context/kv_cache_type.rs b/llama-cpp-bindings/src/context/kv_cache_type.rs index 1648ee370..270469881 100644 --- a/llama-cpp-bindings/src/context/kv_cache_type.rs +++ b/llama-cpp-bindings/src/context/kv_cache_type.rs @@ -167,7 +167,7 @@ mod tests { } #[test] - fn kv_cache_type_all_known_variants_roundtrip() { + fn every_kv_cache_type_converts_to_its_ggml_type_and_back() { let all_variants = [ KvCacheType::F32, KvCacheType::F16, diff --git a/llama-cpp-bindings/src/context/memory.rs b/llama-cpp-bindings/src/context/memory.rs index 9c49cdbec..2fe2757d2 100644 --- a/llama-cpp-bindings/src/context/memory.rs +++ b/llama-cpp-bindings/src/context/memory.rs @@ -3,7 +3,9 @@ use std::ffi::c_int; use std::num::NonZeroU8; use std::ptr; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use crate::context::LlamaContext; use crate::context::memory_of; @@ -41,21 +43,17 @@ fn memory_seq_add_status_to_result( }?; Err(MemorySeqAddError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_CTX_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_memory_seq_add", - detail: "was given a null ctx argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_CTX_ARG => Err(FfiContractError { + operation: "llama_rs_memory_seq_add", + detail: "was given a null ctx argument", } - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_MODEL => { - Err(llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_memory_seq_add", - detail: "was given a null model argument", - } - .into()) + .into()), + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_MODEL => Err(FfiContractError { + operation: "llama_rs_memory_seq_add", + detail: "was given a null model argument", } - other => Err(llama_cpp_ffi_status::FfiStatusError { + .into()), + other => Err(FfiStatusError { operation: "llama_rs_memory_seq_add", code: i64::from(other), } @@ -91,21 +89,17 @@ fn memory_seq_div_status_to_result( }?; Err(MemorySeqDivError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_CTX_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_memory_seq_div", - detail: "was given a null ctx argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_CTX_ARG => Err(FfiContractError { + operation: "llama_rs_memory_seq_div", + detail: "was given a null ctx argument", } - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_MODEL => { - Err(llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_memory_seq_div", - detail: "was given a null model argument", - } - .into()) + .into()), + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_MODEL => Err(FfiContractError { + operation: "llama_rs_memory_seq_div", + detail: "was given a null model argument", } - other => Err(llama_cpp_ffi_status::FfiStatusError { + .into()), + other => Err(FfiStatusError { operation: "llama_rs_memory_seq_div", code: i64::from(other), } @@ -121,22 +115,20 @@ fn memory_seq_pos_max_status_to_result( ) -> Result { match status { llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_OK => Ok(position), - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_CTX_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_memory_seq_pos_max", - detail: "context pointer was null", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_CTX_ARG => Err(FfiContractError { + operation: "llama_rs_memory_seq_pos_max", + detail: "context pointer was null", } + .into()), llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_OUT_POSITION_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_memory_seq_pos_max", detail: "output position pointer was null", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_OUT_ERROR_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_memory_seq_pos_max", detail: "output error pointer was null", } @@ -164,7 +156,7 @@ fn memory_seq_pos_max_status_to_result( }?; Err(MemorySeqPosMaxError::Reported { message }) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_memory_seq_pos_max", code: i64::from(other), } @@ -332,6 +324,9 @@ impl LlamaContext<'_> { mod memory_seq_add_status_to_result_tests { use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::memory_seq_add_status_to_result; use crate::error::memory_seq_add_error::MemorySeqAddError; @@ -385,7 +380,7 @@ mod memory_seq_add_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), ), - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_memory_seq_add", detail: "reported a thrown C++ exception without an error message", } @@ -415,12 +410,10 @@ mod memory_seq_add_status_to_result_tests { assert_eq!( result, - Err(MemorySeqAddError::FfiStatus( - llama_cpp_ffi_status::FfiStatusError { - operation: "llama_rs_memory_seq_add", - code: 255, - } - )) + Err(MemorySeqAddError::FfiStatus(FfiStatusError { + operation: "llama_rs_memory_seq_add", + code: 255, + })) ); } @@ -433,7 +426,7 @@ mod memory_seq_add_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_memory_seq_add", detail: "was given a null ctx argument", } @@ -451,7 +444,7 @@ mod memory_seq_add_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_memory_seq_add", detail: "was given a null model argument", } @@ -474,6 +467,9 @@ mod memory_seq_add_status_to_result_tests { mod memory_seq_div_status_to_result_tests { use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::memory_seq_div_status_to_result; use crate::error::memory_seq_div_error::MemorySeqDivError; @@ -527,7 +523,7 @@ mod memory_seq_div_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), ), - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_memory_seq_div", detail: "reported a thrown C++ exception without an error message", } @@ -557,12 +553,10 @@ mod memory_seq_div_status_to_result_tests { assert_eq!( result, - Err(MemorySeqDivError::FfiStatus( - llama_cpp_ffi_status::FfiStatusError { - operation: "llama_rs_memory_seq_div", - code: 255, - } - )) + Err(MemorySeqDivError::FfiStatus(FfiStatusError { + operation: "llama_rs_memory_seq_div", + code: 255, + })) ); } @@ -575,7 +569,7 @@ mod memory_seq_div_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_memory_seq_div", detail: "was given a null ctx argument", } @@ -593,7 +587,7 @@ mod memory_seq_div_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_memory_seq_div", detail: "was given a null model argument", } @@ -616,6 +610,9 @@ mod memory_seq_div_status_to_result_tests { mod memory_seq_pos_max_status_to_result_tests { use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::memory_seq_pos_max_status_to_result; use crate::error::memory_seq_pos_max_error::MemorySeqPosMaxError; @@ -641,12 +638,10 @@ mod memory_seq_pos_max_status_to_result_tests { 2, ptr::null_mut(), ), - Err(MemorySeqPosMaxError::FfiContract( - llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_memory_seq_pos_max", - detail: "context pointer was null", - } - )) + Err(MemorySeqPosMaxError::FfiContract(FfiContractError { + operation: "llama_rs_memory_seq_pos_max", + detail: "context pointer was null", + })) ); } @@ -659,12 +654,10 @@ mod memory_seq_pos_max_status_to_result_tests { 2, ptr::null_mut(), ), - Err(MemorySeqPosMaxError::FfiContract( - llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_memory_seq_pos_max", - detail: "output position pointer was null", - } - )) + Err(MemorySeqPosMaxError::FfiContract(FfiContractError { + operation: "llama_rs_memory_seq_pos_max", + detail: "output position pointer was null", + })) ); } @@ -677,12 +670,10 @@ mod memory_seq_pos_max_status_to_result_tests { 2, ptr::null_mut(), ), - Err(MemorySeqPosMaxError::FfiContract( - llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_memory_seq_pos_max", - detail: "output error pointer was null", - } - )) + Err(MemorySeqPosMaxError::FfiContract(FfiContractError { + operation: "llama_rs_memory_seq_pos_max", + detail: "output error pointer was null", + })) ); } @@ -734,7 +725,7 @@ mod memory_seq_pos_max_status_to_result_tests { 2, ptr::null_mut(), ), - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_memory_seq_pos_max", detail: "reported a thrown C++ exception without an error message", } @@ -765,12 +756,10 @@ mod memory_seq_pos_max_status_to_result_tests { fn seq_pos_max_unknown_status_is_preserved() { assert_eq!( memory_seq_pos_max_status_to_result(255, -1, 2, ptr::null_mut(),), - Err(MemorySeqPosMaxError::FfiStatus( - llama_cpp_ffi_status::FfiStatusError { - operation: "llama_rs_memory_seq_pos_max", - code: 255, - } - )) + Err(MemorySeqPosMaxError::FfiStatus(FfiStatusError { + operation: "llama_rs_memory_seq_pos_max", + code: 255, + })) ); } diff --git a/llama-cpp-bindings/src/context/session.rs b/llama-cpp-bindings/src/context/session.rs index 09ad084e2..bf0ff1c73 100644 --- a/llama-cpp-bindings/src/context/session.rs +++ b/llama-cpp-bindings/src/context/session.rs @@ -3,6 +3,10 @@ use std::ffi::c_char; use std::path::Path; use std::ptr; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; + use crate::context::LlamaContext; use crate::context::llama_state_seq_flags::LlamaStateSeqFlags; use crate::context::load_seq_state_error::LoadSeqStateError; @@ -21,34 +25,28 @@ fn state_data_status_to_result( ) -> Result { match status { llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_OK => Ok(byte_count), - llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_CTX_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { - operation, - detail: "was given a null ctx argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_CTX_ARG => Err(FfiContractError { + operation, + detail: "was given a null ctx argument", } - llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_BUFFER_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { - operation, - detail: "was given a null buffer argument", - } - .into()) + .into()), + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_BUFFER_ARG => Err(FfiContractError { + operation, + detail: "was given a null buffer argument", } + .into()), llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_OUT_BYTE_COUNT_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation, detail: "was given a null out_byte_count argument", } .into()) } - llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_OUT_ERROR_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { - operation, - detail: "was given a null out_error argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_OUT_ERROR_ARG => Err(FfiContractError { + operation, + detail: "was given a null out_error argument", } + .into()), llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_ERROR_STRING_ALLOCATION_FAILED => { Err(StateDataError::NotEnoughMemory) } @@ -57,7 +55,7 @@ fn state_data_status_to_result( } llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_LLAMA_CPP_THREW_CXX_EXCEPTION => { let message = unsafe { - llama_cpp_ffi_status::read_and_free_cpp_string( + read_and_free_cpp_string( out_error, operation, "reported a thrown C++ exception without an error message", @@ -66,7 +64,7 @@ fn state_data_status_to_result( Err(StateDataError::Reported { message }) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation, code: i64::from(other), } @@ -485,6 +483,9 @@ mod process_seq_load_result_tests { mod state_data_status_to_result_tests { use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::state_data_status_to_result; use crate::context::state_data_error::StateDataError; @@ -522,12 +523,10 @@ mod state_data_status_to_result_tests { ] { assert_eq!( state_data_status_to_result(status, 0, ptr::null_mut(), "llama_rs_state_get_data"), - Err(StateDataError::FfiContract( - llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_state_get_data", - detail, - } - )) + Err(StateDataError::FfiContract(FfiContractError { + operation: "llama_rs_state_get_data", + detail, + })) ); } } @@ -572,12 +571,10 @@ mod state_data_status_to_result_tests { ptr::null_mut(), "llama_rs_state_get_data", ), - Err(StateDataError::FfiContract( - llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_state_get_data", - detail: "reported a thrown C++ exception without an error message", - } - )) + Err(StateDataError::FfiContract(FfiContractError { + operation: "llama_rs_state_get_data", + detail: "reported a thrown C++ exception without an error message", + })) ); } @@ -585,12 +582,10 @@ mod state_data_status_to_result_tests { fn state_data_status_to_result_preserves_an_unknown_status() { assert_eq!( state_data_status_to_result(255, 0, ptr::null_mut(), "llama_rs_state_get_data"), - Err(StateDataError::FfiStatus( - llama_cpp_ffi_status::FfiStatusError { - operation: "llama_rs_state_get_data", - code: 255, - } - )) + Err(StateDataError::FfiStatus(FfiStatusError { + operation: "llama_rs_state_get_data", + code: 255, + })) ); } } diff --git a/llama-cpp-bindings/src/context/state_data_error.rs b/llama-cpp-bindings/src/context/state_data_error.rs index f80ba3b4a..3cd41a60c 100644 --- a/llama-cpp-bindings/src/context/state_data_error.rs +++ b/llama-cpp-bindings/src/context/state_data_error.rs @@ -1,9 +1,12 @@ +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum StateDataError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/error/batch_transfer_error.rs b/llama-cpp-bindings/src/error/batch_transfer_error.rs index 69787146d..535aa5a55 100644 --- a/llama-cpp-bindings/src/error/batch_transfer_error.rs +++ b/llama-cpp-bindings/src/error/batch_transfer_error.rs @@ -1,11 +1,14 @@ +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use crate::context::context_limits::ContextLimits; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum BatchTransferError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("the batch was built for context limits {batch:?}, but the context has {context:?}")] LimitsMismatch { batch: ContextLimits, diff --git a/llama-cpp-bindings/src/error/chat_session_error.rs b/llama-cpp-bindings/src/error/chat_session_error.rs index 30426a380..2221510da 100644 --- a/llama-cpp-bindings/src/error/chat_session_error.rs +++ b/llama-cpp-bindings/src/error/chat_session_error.rs @@ -1,6 +1,9 @@ use std::ffi::NulError; use std::str::Utf8Error; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use crate::error::chat_templates_error::ChatTemplatesError; use crate::error::grammar_error::GrammarError; use crate::error::sampler_accept_error::SamplerAcceptError; @@ -9,9 +12,9 @@ use crate::error::string_to_token_error::StringToTokenError; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum ChatSessionError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("the model's chat templates are unavailable: {0}")] Templates(#[from] ChatTemplatesError), #[error("a chat message is not valid UTF-8")] diff --git a/llama-cpp-bindings/src/error/chat_templates_error.rs b/llama-cpp-bindings/src/error/chat_templates_error.rs index ac3cdada9..539949e33 100644 --- a/llama-cpp-bindings/src/error/chat_templates_error.rs +++ b/llama-cpp-bindings/src/error/chat_templates_error.rs @@ -1,9 +1,12 @@ +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum ChatTemplatesError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("the model has no chat template")] ModelHasNoChatTemplate, #[error("not enough memory")] diff --git a/llama-cpp-bindings/src/error/embeddings_error.rs b/llama-cpp-bindings/src/error/embeddings_error.rs index db7e8b175..f9d2f3ed2 100644 --- a/llama-cpp-bindings/src/error/embeddings_error.rs +++ b/llama-cpp-bindings/src/error/embeddings_error.rs @@ -1,3 +1,6 @@ +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum EmbeddingsError { #[error("Embeddings weren't enabled in the context options")] @@ -41,9 +44,9 @@ pub enum EmbeddingsError { #[error("llama.cpp failed while checking for a NextN hidden state: {message}")] NextnProbeReported { message: String }, #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error( "No NextN embedding exists for token {token_index}; it was not marked as an output of the last decoded batch" )] diff --git a/llama-cpp-bindings/src/error/fit_error.rs b/llama-cpp-bindings/src/error/fit_error.rs index 7c45a6a05..341bff51f 100644 --- a/llama-cpp-bindings/src/error/fit_error.rs +++ b/llama-cpp-bindings/src/error/fit_error.rs @@ -1,9 +1,12 @@ +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] pub enum FitError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("no parameter combination fits available memory")] NoFittingMemoryLayout, #[error("parameter fitting aborted")] diff --git a/llama-cpp-bindings/src/error/grammar_error.rs b/llama-cpp-bindings/src/error/grammar_error.rs index 22e2babde..68481c528 100644 --- a/llama-cpp-bindings/src/error/grammar_error.rs +++ b/llama-cpp-bindings/src/error/grammar_error.rs @@ -1,14 +1,17 @@ use std::ffi::NulError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use crate::error::sampling_error::SamplingError; use crate::error::token_to_string_error::TokenToStringError; #[derive(Debug, thiserror::Error, PartialEq, Eq)] pub enum GrammarError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("sampler initialization failed: {0}")] SamplerInitialization(#[from] SamplingError), #[error("the approximate token environment could not be built: {0}")] diff --git a/llama-cpp-bindings/src/error/json_schema_to_grammar_error.rs b/llama-cpp-bindings/src/error/json_schema_to_grammar_error.rs index 5618ca78e..be4aa8373 100644 --- a/llama-cpp-bindings/src/error/json_schema_to_grammar_error.rs +++ b/llama-cpp-bindings/src/error/json_schema_to_grammar_error.rs @@ -1,12 +1,15 @@ use std::ffi::NulError; use std::string::FromUtf8Error; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, PartialEq, Eq, thiserror::Error)] pub enum JsonSchemaToGrammarError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("schema string contains an interior NUL byte: {0}")] SchemaContainsNulByte(#[from] NulError), #[error("JSON schema is invalid: {message}")] diff --git a/llama-cpp-bindings/src/error/llama_context_load_error.rs b/llama-cpp-bindings/src/error/llama_context_load_error.rs index b571d7aae..a112b096a 100644 --- a/llama-cpp-bindings/src/error/llama_context_load_error.rs +++ b/llama-cpp-bindings/src/error/llama_context_load_error.rs @@ -1,9 +1,12 @@ +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum LlamaContextLoadError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("context could not be constructed")] Unconstructible, #[error("not enough memory")] diff --git a/llama-cpp-bindings/src/error/llama_model_load_error.rs b/llama-cpp-bindings/src/error/llama_model_load_error.rs index b9f69b616..7f9081996 100644 --- a/llama-cpp-bindings/src/error/llama_model_load_error.rs +++ b/llama-cpp-bindings/src/error/llama_model_load_error.rs @@ -1,12 +1,15 @@ use std::ffi::NulError; use std::path::PathBuf; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, PartialEq, Eq, thiserror::Error)] pub enum LlamaModelLoadError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("null byte in string {0}")] NullError(#[from] NulError), #[error("failed to convert path {0} to str")] diff --git a/llama-cpp-bindings/src/error/memory_seq_add_error.rs b/llama-cpp-bindings/src/error/memory_seq_add_error.rs index 0f6aa2b47..ca936c92d 100644 --- a/llama-cpp-bindings/src/error/memory_seq_add_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_add_error.rs @@ -1,11 +1,14 @@ use std::num::TryFromIntError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum MemorySeqAddError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("provided start position is too large for an i32")] P0TooLarge(#[source] TryFromIntError), #[error("provided end position is too large for an i32")] diff --git a/llama-cpp-bindings/src/error/memory_seq_div_error.rs b/llama-cpp-bindings/src/error/memory_seq_div_error.rs index e4b35ed03..87f26b9b3 100644 --- a/llama-cpp-bindings/src/error/memory_seq_div_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_div_error.rs @@ -1,11 +1,14 @@ use std::num::TryFromIntError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum MemorySeqDivError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("provided start position is too large for an i32")] P0TooLarge(#[source] TryFromIntError), #[error("provided end position is too large for an i32")] diff --git a/llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs b/llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs index eed84ba53..795d02744 100644 --- a/llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs @@ -1,9 +1,12 @@ +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum MemorySeqPosMaxError { #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error("context has no memory module available")] MemoryHandleUnavailable, #[error("sequence id {seq_id} is outside the context sequence range")] diff --git a/llama-cpp-bindings/src/error/parse_chat_message_error.rs b/llama-cpp-bindings/src/error/parse_chat_message_error.rs index d3c4fb2c4..5dae9bccc 100644 --- a/llama-cpp-bindings/src/error/parse_chat_message_error.rs +++ b/llama-cpp-bindings/src/error/parse_chat_message_error.rs @@ -1,11 +1,14 @@ use std::str::Utf8Error; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum ParseChatMessageError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/error/process_failure.rs b/llama-cpp-bindings/src/error/process_failure.rs index b557ef4c0..9433b598c 100644 --- a/llama-cpp-bindings/src/error/process_failure.rs +++ b/llama-cpp-bindings/src/error/process_failure.rs @@ -1,10 +1,13 @@ +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + /// How `llama_rs_process` failed before llama.cpp returned a code of its own. #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum ProcessFailure { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/error/sample_error.rs b/llama-cpp-bindings/src/error/sample_error.rs index b9467080e..7be976fb1 100644 --- a/llama-cpp-bindings/src/error/sample_error.rs +++ b/llama-cpp-bindings/src/error/sample_error.rs @@ -1,3 +1,6 @@ +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use crate::error::ingest_error::IngestError; use crate::error::logits_error::LogitsError; use crate::error::sampler_apply_error::SamplerApplyError; @@ -5,9 +8,9 @@ use crate::error::sampler_apply_error::SamplerApplyError; #[derive(Debug, thiserror::Error, PartialEq, Eq)] pub enum SampleError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("the logits to sample from are unavailable: {0}")] Logits(#[from] LogitsError), #[error("not enough memory")] diff --git a/llama-cpp-bindings/src/error/sampler_accept_error.rs b/llama-cpp-bindings/src/error/sampler_accept_error.rs index 81b68f310..91331b6a9 100644 --- a/llama-cpp-bindings/src/error/sampler_accept_error.rs +++ b/llama-cpp-bindings/src/error/sampler_accept_error.rs @@ -1,9 +1,12 @@ +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, PartialEq, Eq, thiserror::Error)] pub enum SamplerAcceptError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/error/sampler_apply_error.rs b/llama-cpp-bindings/src/error/sampler_apply_error.rs index 4cab8977f..d7ed80125 100644 --- a/llama-cpp-bindings/src/error/sampler_apply_error.rs +++ b/llama-cpp-bindings/src/error/sampler_apply_error.rs @@ -1,9 +1,12 @@ +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, thiserror::Error, Clone, PartialEq, Eq)] pub enum SamplerApplyError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("the sampler pointer was null when applying to the token data array")] NullSampler, #[error("the sampler excluded every candidate, leaving no token to select")] diff --git a/llama-cpp-bindings/src/error/sampling_error.rs b/llama-cpp-bindings/src/error/sampling_error.rs index c22991e31..6b70a3229 100644 --- a/llama-cpp-bindings/src/error/sampling_error.rs +++ b/llama-cpp-bindings/src/error/sampling_error.rs @@ -1,11 +1,14 @@ use std::ffi::NulError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum SamplingError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("a value does not fit into i32")] IntegerOverflow(#[source] std::num::TryFromIntError), #[error("the llama.cpp library ran out of memory initializing the {sampler} sampler")] diff --git a/llama-cpp-bindings/src/error/string_to_token_error.rs b/llama-cpp-bindings/src/error/string_to_token_error.rs index 41ed7a5e0..93d55a66f 100644 --- a/llama-cpp-bindings/src/error/string_to_token_error.rs +++ b/llama-cpp-bindings/src/error/string_to_token_error.rs @@ -1,11 +1,14 @@ use std::ffi::NulError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, PartialEq, Eq, thiserror::Error)] pub enum StringToTokenError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("{0}")] NulError(#[from] NulError), #[error("{0}")] diff --git a/llama-cpp-bindings/src/json_schema_to_grammar.rs b/llama-cpp-bindings/src/json_schema_to_grammar.rs index 2526d5303..334c08118 100644 --- a/llama-cpp-bindings/src/json_schema_to_grammar.rs +++ b/llama-cpp-bindings/src/json_schema_to_grammar.rs @@ -1,7 +1,9 @@ use std::ffi::{CStr, CString, c_char}; use std::ptr; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use crate::error::json_schema_to_grammar_error::JsonSchemaToGrammarError; @@ -19,7 +21,7 @@ unsafe fn json_schema_to_grammar_status_to_result( match status { llama_cpp_bindings_sys::LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_OK => { if out.is_null() { - return Err(llama_cpp_ffi_status::FfiContractError { + return Err(FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "success status contained a null grammar string", } @@ -66,27 +68,27 @@ unsafe fn json_schema_to_grammar_status_to_result( Err(JsonSchemaToGrammarError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_NULL_SCHEMA_JSON_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "was given a null schema_json argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_NULL_OUT_GRAMMAR_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "was given a null out_grammar argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_NULL_OUT_ERROR_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "was given a null out_error argument", } .into()) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_json_schema_to_grammar", code: i64::from(other), } @@ -123,7 +125,7 @@ mod tests { use crate::json_schema_to_grammar::json_schema_to_grammar; #[test] - fn simple_object() { + fn an_object_schema_converts_to_a_grammar() { let schema = r#"{"type": "object", "properties": {"name": {"type": "string"}}}"#; let grammar = json_schema_to_grammar(schema).expect("schema converts to grammar"); @@ -142,7 +144,7 @@ mod tests { } #[test] - fn simple_string() { + fn a_string_schema_converts_to_a_grammar() { let schema = r#"{"type": "string"}"#; let grammar = json_schema_to_grammar(schema).expect("schema converts to grammar"); @@ -178,6 +180,9 @@ mod json_schema_to_grammar_status_to_result_tests { use std::mem::discriminant; use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::json_schema_to_grammar_status_to_result; use crate::error::json_schema_to_grammar_error::JsonSchemaToGrammarError; @@ -193,7 +198,7 @@ mod json_schema_to_grammar_status_to_result_tests { assert_eq!( result, - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "reported a thrown C++ exception without an error message", } @@ -213,7 +218,7 @@ mod json_schema_to_grammar_status_to_result_tests { assert_eq!( result, - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "reported a thrown C++ exception without an error message", } @@ -233,7 +238,7 @@ mod json_schema_to_grammar_status_to_result_tests { assert_eq!( result, - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "reported a thrown C++ exception without an error message", } @@ -332,12 +337,10 @@ mod json_schema_to_grammar_status_to_result_tests { assert_eq!( result, - Err(JsonSchemaToGrammarError::FfiContract( - llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_json_schema_to_grammar", - detail: "success status contained a null grammar string", - } - )) + Err(JsonSchemaToGrammarError::FfiContract(FfiContractError { + operation: "llama_rs_json_schema_to_grammar", + detail: "success status contained a null grammar string", + })) ); } @@ -349,12 +352,10 @@ mod json_schema_to_grammar_status_to_result_tests { assert_eq!( result, - Err(JsonSchemaToGrammarError::FfiStatus( - llama_cpp_ffi_status::FfiStatusError { - operation: "llama_rs_json_schema_to_grammar", - code: 255, - } - )) + Err(JsonSchemaToGrammarError::FfiStatus(FfiStatusError { + operation: "llama_rs_json_schema_to_grammar", + code: 255, + })) ); } @@ -370,7 +371,7 @@ mod json_schema_to_grammar_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "was given a null schema_json argument", } @@ -391,7 +392,7 @@ mod json_schema_to_grammar_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "was given a null out_grammar argument", } @@ -412,7 +413,7 @@ mod json_schema_to_grammar_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "was given a null out_error argument", } diff --git a/llama-cpp-bindings/src/lazy_grammar_triggers.rs b/llama-cpp-bindings/src/lazy_grammar_triggers.rs index 11b1584d0..d9a87563f 100644 --- a/llama-cpp-bindings/src/lazy_grammar_triggers.rs +++ b/llama-cpp-bindings/src/lazy_grammar_triggers.rs @@ -1,7 +1,9 @@ use std::ffi::c_char; use std::ptr; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use crate::chat_grammar_trigger::ChatGrammarTrigger; use crate::error::grammar_error::GrammarError; @@ -50,28 +52,24 @@ fn regex_escape_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_OUT_OF_MEMORY => { Err(GrammarError::LlamaCppOutOfMemory) } - llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_TEXT_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { - operation: OPERATION, - detail: "was given a null text argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_TEXT_ARG => Err(FfiContractError { + operation: OPERATION, + detail: "was given a null text argument", } + .into()), llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_OUT_STRING_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: OPERATION, detail: "was given a null out_string argument", } .into()) } - llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_OUT_ERROR_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { - operation: OPERATION, - detail: "was given a null out_error argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_OUT_ERROR_ARG => Err(FfiContractError { + operation: OPERATION, + detail: "was given a null out_error argument", } - other => Err(llama_cpp_ffi_status::FfiStatusError { + .into()), + other => Err(FfiStatusError { operation: OPERATION, code: i64::from(other), } @@ -129,8 +127,10 @@ impl LazyGrammarTriggers { mod tests { use std::ptr; - use llama_cpp_fault_injection::ArmedFault; - use llama_cpp_fault_injection::injected_fault_message; + use llama_cpp_fault_injection::armed_fault::ArmedFault; + use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; use super::LazyGrammarTriggers; use super::anchored_pattern; @@ -145,12 +145,10 @@ mod tests { regex_escape_status_to_result(status, out_string, out_error) }; let contract = |detail| { - Err(GrammarError::FfiContract( - llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_regex_escape", - detail, - }, - )) + Err(GrammarError::FfiContract(FfiContractError { + operation: "llama_rs_regex_escape", + detail, + })) }; assert_eq!( @@ -210,12 +208,10 @@ mod tests { contract("was given a null text argument"), contract("was given a null out_string argument"), contract("was given a null out_error argument"), - Err(GrammarError::FfiStatus( - llama_cpp_ffi_status::FfiStatusError { - operation: "llama_rs_regex_escape", - code: 255 - } - )), + Err(GrammarError::FfiStatus(FfiStatusError { + operation: "llama_rs_regex_escape", + code: 255 + })), ] ); } @@ -228,12 +224,10 @@ mod tests { ptr::null_mut(), ptr::null_mut() ), - Err(GrammarError::FfiContract( - llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_regex_escape", - detail: "reported a thrown C++ exception without an error message" - } - )) + Err(GrammarError::FfiContract(FfiContractError { + operation: "llama_rs_regex_escape", + detail: "reported a thrown C++ exception without an error message" + })) ); } diff --git a/llama-cpp-bindings/src/llguidance_sampler.rs b/llama-cpp-bindings/src/llguidance_sampler.rs index 396a9fbad..ffe9d2160 100644 --- a/llama-cpp-bindings/src/llguidance_sampler.rs +++ b/llama-cpp-bindings/src/llguidance_sampler.rs @@ -9,8 +9,8 @@ use llguidance::earley::SlicedBiasComputer; use toktrie::ApproximateTokEnv; use toktrie::InferenceCapabilities; -use llama_cpp_error_recorder::RecordedError; -use llama_cpp_error_recorder::record; +use llama_cpp_error_recorder::record::record; +use llama_cpp_error_recorder::recorded_error::RecordedError; use crate::error::grammar_error::GrammarError; use crate::grammar_matcher::GrammarMatcher; @@ -200,7 +200,7 @@ mod tests { use toktrie::TokRxInfo; use toktrie::TokTrie; - use llama_cpp_fault_injection::ArmedFault; + use llama_cpp_fault_injection::armed_fault::ArmedFault; use super::UNCOMPILABLE_BIAS_SLICE; use super::create_llg_sampler_from_tok_env; diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index ddcdbdff1..c1f838832 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -37,7 +37,9 @@ use toktrie::INVALID_TOKEN; use toktrie::TokRxInfo; use toktrie::TokTrie; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use crate::chat_templates::ChatTemplates; use crate::error::chat_template_error::ChatTemplateError; @@ -141,7 +143,7 @@ unsafe fn load_model_from_file_status_to_result( match status { llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_OK => { let model = NonNull::new(out_model).ok_or_else(|| { - LlamaModelLoadError::from(llama_cpp_ffi_status::FfiContractError { + LlamaModelLoadError::from(FfiContractError { operation: "llama_rs_load_model_from_file", detail: "success status contained a null model", }) @@ -180,27 +182,27 @@ unsafe fn load_model_from_file_status_to_result( Err(LlamaModelLoadError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_NULL_PATH_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_load_model_from_file", detail: "was given a null path argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_NULL_OUT_MODEL_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_load_model_from_file", detail: "was given a null out_model argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_NULL_OUT_ERROR_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_load_model_from_file", detail: "was given a null out_error argument", } .into()) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_load_model_from_file", code: i64::from(other), } @@ -662,7 +664,7 @@ fn initialize_lora_adapter<'model>( initialize: impl FnOnce() -> *mut llama_cpp_bindings_sys::llama_adapter_lora, ) -> Result, LlamaLoraAdapterInitError> { let adapter = NonNull::new(initialize()).ok_or(LlamaLoraAdapterInitError::Unloadable)?; - Ok(LlamaLoraAdapter::new(adapter)) + Ok(unsafe { LlamaLoraAdapter::new(adapter) }) } impl LlamaModel { @@ -767,7 +769,7 @@ unsafe fn tokenize_status_to_result( } llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_VOCAB_ARG => { unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_tokenize", detail: "was given a null vocab argument", } @@ -775,7 +777,7 @@ unsafe fn tokenize_status_to_result( } llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_TEXT_ARG => { unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_tokenize", detail: "was given a null text argument", } @@ -783,7 +785,7 @@ unsafe fn tokenize_status_to_result( } llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_OUT_RETURNED_COUNT_ARG => { unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_tokenize", detail: "was given a null out_returned_count argument", } @@ -791,7 +793,7 @@ unsafe fn tokenize_status_to_result( } llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_OUT_ERROR_ARG => { unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_tokenize", detail: "was given a null out_error argument", } @@ -799,7 +801,7 @@ unsafe fn tokenize_status_to_result( } other => { unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(llama_cpp_ffi_status::FfiStatusError { + Err(FfiStatusError { operation: "llama_rs_tokenize", code: i64::from(other), } @@ -1191,6 +1193,9 @@ mod load_model_from_file_status_to_result_tests { use std::path::Path; use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::load_model_from_file_status_to_result; use crate::error::llama_model_load_error::LlamaModelLoadError; @@ -1207,7 +1212,7 @@ mod load_model_from_file_status_to_result_tests { assert_eq!( result.unwrap_err(), - LlamaModelLoadError::FfiContract(llama_cpp_ffi_status::FfiContractError { + LlamaModelLoadError::FfiContract(FfiContractError { operation: "llama_rs_load_model_from_file", detail: "success status contained a null model", }) @@ -1281,7 +1286,7 @@ mod load_model_from_file_status_to_result_tests { assert_eq!( result.unwrap_err(), - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_load_model_from_file", detail: "reported a thrown C++ exception without an error message", } @@ -1302,7 +1307,7 @@ mod load_model_from_file_status_to_result_tests { assert_eq!( result.unwrap_err(), - LlamaModelLoadError::FfiStatus(llama_cpp_ffi_status::FfiStatusError { + LlamaModelLoadError::FfiStatus(FfiStatusError { operation: "llama_rs_load_model_from_file", code: 255, }) @@ -1322,7 +1327,7 @@ mod load_model_from_file_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_load_model_from_file", detail: "was given a null path argument", } @@ -1344,7 +1349,7 @@ mod load_model_from_file_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_load_model_from_file", detail: "was given a null out_model argument", } @@ -1366,7 +1371,7 @@ mod load_model_from_file_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_load_model_from_file", detail: "was given a null out_error argument", } @@ -1396,6 +1401,9 @@ mod load_model_from_file_status_to_result_tests { mod tokenize_status_to_result_tests { use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::tokenize_status_to_result; use crate::error::string_to_token_error::StringToTokenError; @@ -1457,7 +1465,7 @@ mod tokenize_status_to_result_tests { assert_eq!( result, - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_tokenize", detail: "reported a thrown C++ exception without an error message", } @@ -1471,12 +1479,10 @@ mod tokenize_status_to_result_tests { assert_eq!( result, - Err(StringToTokenError::FfiStatus( - llama_cpp_ffi_status::FfiStatusError { - operation: "llama_rs_tokenize", - code: 255, - } - )) + Err(StringToTokenError::FfiStatus(FfiStatusError { + operation: "llama_rs_tokenize", + code: 255, + })) ); } @@ -1492,7 +1498,7 @@ mod tokenize_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_tokenize", detail: "was given a null vocab argument", } @@ -1513,7 +1519,7 @@ mod tokenize_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_tokenize", detail: "was given a null text argument", } @@ -1534,7 +1540,7 @@ mod tokenize_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_tokenize", detail: "was given a null out_returned_count argument", } @@ -1555,7 +1561,7 @@ mod tokenize_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_tokenize", detail: "was given a null out_error argument", } diff --git a/llama-cpp-bindings/src/model/llama_chat_message.rs b/llama-cpp-bindings/src/model/llama_chat_message.rs index 6a1ca8dfd..b64b91df3 100644 --- a/llama-cpp-bindings/src/model/llama_chat_message.rs +++ b/llama-cpp-bindings/src/model/llama_chat_message.rs @@ -24,7 +24,7 @@ mod tests { use crate::model::llama_chat_message::LlamaChatMessage; #[test] - fn valid_construction() { + fn a_role_and_content_without_nul_bytes_form_a_message() { let message = LlamaChatMessage::new("user".to_string(), "hello".to_string()); assert!(message.is_ok()); diff --git a/llama-cpp-bindings/src/model/llama_chat_template.rs b/llama-cpp-bindings/src/model/llama_chat_template.rs index 06d79a6f3..ac4f82337 100644 --- a/llama-cpp-bindings/src/model/llama_chat_template.rs +++ b/llama-cpp-bindings/src/model/llama_chat_template.rs @@ -59,7 +59,7 @@ mod tests { } #[test] - fn debug_formatting() { + fn debug_formats_a_template_as_its_quoted_name() { let template = LlamaChatTemplate::new("chatml").unwrap(); let debug_output = format!("{template:?}"); diff --git a/llama-cpp-bindings/src/model/llama_lora_adapter.rs b/llama-cpp-bindings/src/model/llama_lora_adapter.rs index 0517a20b9..fc3f17b6f 100644 --- a/llama-cpp-bindings/src/model/llama_lora_adapter.rs +++ b/llama-cpp-bindings/src/model/llama_lora_adapter.rs @@ -14,7 +14,14 @@ pub struct LlamaLoraAdapter<'model> { } impl LlamaLoraAdapter<'_> { - pub(crate) const fn new( + /// Takes ownership of an adapter llama.cpp initialized, and frees it on drop. + /// + /// # Safety + /// + /// `lora_adapter` must come from `llama_adapter_lora_init` for the model this adapter + /// borrows, and no other [`LlamaLoraAdapter`] may own it. + #[must_use] + pub const unsafe fn new( lora_adapter: NonNull, ) -> Self { Self { @@ -23,7 +30,8 @@ impl LlamaLoraAdapter<'_> { } } - pub(crate) const fn as_ptr(&self) -> *mut llama_cpp_bindings_sys::llama_adapter_lora { + #[must_use] + pub const fn as_ptr(&self) -> *mut llama_cpp_bindings_sys::llama_adapter_lora { self.lora_adapter } } @@ -46,7 +54,7 @@ mod ownership_tests { #[test] fn adapter_preserves_the_owned_native_pointer() { let pointer = NonNull::dangling(); - let adapter = ManuallyDrop::new(LlamaLoraAdapter::new(pointer)); + let adapter = ManuallyDrop::new(unsafe { LlamaLoraAdapter::new(pointer) }); assert_eq!(adapter.as_ptr(), pointer.as_ptr()); } diff --git a/llama-cpp-bindings/src/model/params.rs b/llama-cpp-bindings/src/model/params.rs index 10cf93546..586b806ec 100644 --- a/llama-cpp-bindings/src/model/params.rs +++ b/llama-cpp-bindings/src/model/params.rs @@ -5,7 +5,9 @@ use std::pin::Pin; use std::ptr; use std::ptr::null; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use crate::backend_device_index::backend_device_index; use crate::context::params::LlamaContextParams; @@ -67,63 +69,55 @@ fn fit_params_status_to_result( }?; Err(FitError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_PATH_MODEL_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_fit_params", - detail: "was given a null path_model argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_PATH_MODEL_ARG => Err(FfiContractError { + operation: "llama_rs_fit_params", + detail: "was given a null path_model argument", } - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_MPARAMS_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_fit_params", - detail: "was given a null mparams argument", - } - .into()) + .into()), + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_MPARAMS_ARG => Err(FfiContractError { + operation: "llama_rs_fit_params", + detail: "was given a null mparams argument", } - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_CPARAMS_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_fit_params", - detail: "was given a null cparams argument", - } - .into()) + .into()), + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_CPARAMS_ARG => Err(FfiContractError { + operation: "llama_rs_fit_params", + detail: "was given a null cparams argument", } + .into()), llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_OUT_UNRECOGNIZED_STATUS_CODE_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_fit_params", detail: "was given a null out_unrecognized_status_code argument", } .into()) } - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_OUT_ERROR_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_fit_params", - detail: "was given a null out_error argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_OUT_ERROR_ARG => Err(FfiContractError { + operation: "llama_rs_fit_params", + detail: "was given a null out_error argument", } + .into()), llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_PATH_MODEL_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_fit_params", detail: "was given an extra model with a null path_model argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_MPARAMS_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_fit_params", detail: "was given an extra model with a null mparams argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_CPARAMS_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_fit_params", detail: "was given an extra model with a null cparams argument", } .into()) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_fit_params", code: i64::from(other), } @@ -495,8 +489,11 @@ mod tests { use std::ffi::CStr; use std::ffi::CString; use std::mem::discriminant; + use std::pin::pin; use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use super::LLAMA_CPP_MAX_DEVICES; use super::LlamaModelParams; use super::fit_params_status_to_result; @@ -521,7 +518,7 @@ mod tests { } fn contract_error_with(detail: &'static str) -> FitError { - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_fit_params", detail, } @@ -724,7 +721,7 @@ mod tests { } #[test] - fn with_devices_empty_list_succeeds() { + fn with_devices_accepts_an_empty_device_list() { let params = LlamaModelParams::default().with_devices(&[]); assert!(params.is_ok()); @@ -732,8 +729,8 @@ mod tests { } #[test] - fn add_cpu_buft_override_succeeds() { - let mut params = std::pin::pin!(LlamaModelParams::default()); + fn add_cpu_buft_override_fills_an_empty_override_slot() { + let mut params = pin!(LlamaModelParams::default()); let result = params.as_mut().add_cpu_buft_override(c"test_pattern"); assert!(result.is_ok()); @@ -742,7 +739,7 @@ mod tests { #[test] fn a_buft_override_pattern_stays_valid_after_the_callers_buffer_changes() { let mut caller_buffer = b"blk\\.0\\.ffn\0".to_vec(); - let mut params = std::pin::pin!(LlamaModelParams::default()); + let mut params = pin!(LlamaModelParams::default()); params .as_mut() @@ -758,7 +755,7 @@ mod tests { #[test] fn add_cpu_buft_override_twice_fails_with_slot_not_empty() { - let mut params = std::pin::pin!(LlamaModelParams::default()); + let mut params = pin!(LlamaModelParams::default()); params .as_mut() .add_cpu_buft_override(c"first_pattern") @@ -769,8 +766,8 @@ mod tests { } #[test] - fn add_cpu_moe_override_succeeds() { - let mut params = std::pin::pin!(LlamaModelParams::default()); + fn add_cpu_moe_override_fills_an_empty_override_slot() { + let mut params = pin!(LlamaModelParams::default()); let result = params.as_mut().add_cpu_moe_override(); assert!(result.is_ok()); @@ -778,7 +775,7 @@ mod tests { #[test] fn append_kv_override_twice_fails_with_slot_not_empty() { - let mut params = std::pin::pin!(LlamaModelParams::default()); + let mut params = pin!(LlamaModelParams::default()); let key = CString::new("first_key").unwrap(); params .as_mut() @@ -840,7 +837,7 @@ mod tests { fn append_kv_override_with_high_byte_returns_invalid_character_error() { let key_bytes: &[u8] = b"\xff\0"; let key = CStr::from_bytes_with_nul(key_bytes).unwrap(); - let mut params = std::pin::pin!(LlamaModelParams::default()); + let mut params = pin!(LlamaModelParams::default()); let result = params .as_mut() .append_kv_override(key, ParamOverrideValue::Int(1)); @@ -858,7 +855,7 @@ mod tests { fn add_cpu_buft_override_with_high_byte_returns_invalid_character_error() { let key_bytes: &[u8] = b"\xff\0"; let key = CStr::from_bytes_with_nul(key_bytes).unwrap(); - let mut params = std::pin::pin!(LlamaModelParams::default()); + let mut params = pin!(LlamaModelParams::default()); let result = params.as_mut().add_cpu_buft_override(key); assert_eq!( @@ -874,7 +871,7 @@ mod tests { fn append_kv_override_with_empty_slot_vector_returns_no_available_slot() { let mut params = LlamaModelParams::default(); params.kv_overrides.clear(); - let mut pinned = std::pin::pin!(params); + let mut pinned = pin!(params); let result = pinned .as_mut() @@ -887,7 +884,7 @@ mod tests { fn add_cpu_buft_override_with_empty_slot_vector_returns_no_available_slot() { let mut params = LlamaModelParams::default(); params.buft_overrides.clear(); - let mut pinned = std::pin::pin!(params); + let mut pinned = pin!(params); let result = pinned.as_mut().add_cpu_buft_override(c"any_pattern"); @@ -898,7 +895,7 @@ mod tests { #[serial_test::serial] fn fit_params_invalid_model_path_returns_error() { let _backend = LlamaBackend::init(); - let mut params = std::pin::pin!(LlamaModelParams::default()); + let mut params = pin!(LlamaModelParams::default()); let mut context_params = LlamaContextParams::default(); let mut margins = vec![0usize; max_devices()]; @@ -917,7 +914,7 @@ mod tests { #[test] fn fit_params_refuses_margins_not_sized_to_the_device_count() { - let mut params = std::pin::pin!(LlamaModelParams::default()); + let mut params = pin!(LlamaModelParams::default()); let mut context_params = LlamaContextParams::default(); let mut margins = vec![0usize; max_devices() + 1]; @@ -1032,6 +1029,9 @@ mod tests { mod fit_params_status_to_result_tests { use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::fit_params_status_to_result; use crate::error::fit_error::FitError; @@ -1118,7 +1118,7 @@ mod fit_params_status_to_result_tests { assert_eq!( result, - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_fit_params", detail: "reported a thrown C++ exception without an error message", } @@ -1132,7 +1132,7 @@ mod fit_params_status_to_result_tests { assert_eq!( result, - Err(FitError::FfiStatus(llama_cpp_ffi_status::FfiStatusError { + Err(FitError::FfiStatus(FfiStatusError { operation: "llama_rs_fit_params", code: 255, })) diff --git a/llama-cpp-bindings/src/model/params/param_override_value.rs b/llama-cpp-bindings/src/model/params/param_override_value.rs index b6765ef25..8bcbb87e5 100644 --- a/llama-cpp-bindings/src/model/params/param_override_value.rs +++ b/llama-cpp-bindings/src/model/params/param_override_value.rs @@ -75,47 +75,26 @@ mod tests { use super::ParamOverrideValue; #[test] - fn tag_bool() { - let value = ParamOverrideValue::Bool(true); - - assert_eq!( - value.tag(), - llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_BOOL - ); - } - - #[test] - fn tag_float() { - let value = ParamOverrideValue::Float(1.23); - - assert_eq!( - value.tag(), - llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_FLOAT - ); - } - - #[test] - fn tag_int() { - let value = ParamOverrideValue::Int(42); - - assert_eq!( - value.tag(), - llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_INT - ); - } - - #[test] - fn tag_str() { - let value = ParamOverrideValue::Str([0; 128]); - + fn each_override_value_is_tagged_with_its_llama_cpp_type() { assert_eq!( - value.tag(), - llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_STR + [ + ParamOverrideValue::Bool(true), + ParamOverrideValue::Float(1.23), + ParamOverrideValue::Int(42), + ParamOverrideValue::Str([0; 128]), + ] + .map(|value| value.tag()), + [ + llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_BOOL, + llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_FLOAT, + llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_INT, + llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_STR, + ] ); } #[test] - fn value_bool_roundtrip() { + fn a_bool_override_stores_its_value_in_the_bool_field() { let value = ParamOverrideValue::Bool(true); let ffi_value = value.value(); let result = unsafe { ffi_value.val_bool }; @@ -124,7 +103,7 @@ mod tests { } #[test] - fn value_float_roundtrip() { + fn a_float_override_stores_its_value_in_the_f64_field() { let value = ParamOverrideValue::Float(1.23); let ffi_value = value.value(); let result = unsafe { ffi_value.val_f64 }; @@ -133,7 +112,7 @@ mod tests { } #[test] - fn value_int_roundtrip() { + fn an_int_override_stores_its_value_in_the_i64_field() { let value = ParamOverrideValue::Int(99); let ffi_value = value.value(); let result = unsafe { ffi_value.val_i64 }; @@ -187,7 +166,7 @@ mod tests { } #[test] - fn value_str_roundtrip() { + fn a_str_override_stores_its_bytes_in_the_str_field() { let mut str_data = [0i8; 128]; str_data[0] = b'h'.cast_signed(); str_data[1] = b'i'.cast_signed(); diff --git a/llama-cpp-bindings/src/model/split_mode.rs b/llama-cpp-bindings/src/model/split_mode.rs index e1c359df9..a8e831eda 100644 --- a/llama-cpp-bindings/src/model/split_mode.rs +++ b/llama-cpp-bindings/src/model/split_mode.rs @@ -52,48 +52,27 @@ mod tests { } #[test] - fn try_from_none_roundtrip() { - let mode = LlamaSplitMode::try_from(llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_NONE).unwrap(); + fn every_llama_cpp_split_mode_converts_to_its_variant_and_back() { + let llama_cpp_split_modes = [ + llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_NONE, + llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_LAYER, + llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_ROW, + llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_TENSOR, + ]; + let modes = llama_cpp_split_modes.map(LlamaSplitMode::try_from); - assert_eq!(mode, LlamaSplitMode::None); assert_eq!( - llama_cpp_bindings_sys::llama_split_mode::from(mode), - llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_NONE + modes, + [ + Ok(LlamaSplitMode::None), + Ok(LlamaSplitMode::Layer), + Ok(LlamaSplitMode::Row), + Ok(LlamaSplitMode::Tensor), + ] ); - } - - #[test] - fn try_from_layer_roundtrip() { - let mode = - LlamaSplitMode::try_from(llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_LAYER).unwrap(); - - assert_eq!(mode, LlamaSplitMode::Layer); - assert_eq!( - llama_cpp_bindings_sys::llama_split_mode::from(mode), - llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_LAYER - ); - } - - #[test] - fn try_from_row_roundtrip() { - let mode = LlamaSplitMode::try_from(llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_ROW).unwrap(); - - assert_eq!(mode, LlamaSplitMode::Row); - assert_eq!( - llama_cpp_bindings_sys::llama_split_mode::from(mode), - llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_ROW - ); - } - - #[test] - fn try_from_tensor_roundtrip() { - let mode = - LlamaSplitMode::try_from(llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_TENSOR).unwrap(); - - assert_eq!(mode, LlamaSplitMode::Tensor); assert_eq!( - llama_cpp_bindings_sys::llama_split_mode::from(mode), - llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_TENSOR + modes.map(|mode| mode.map(llama_cpp_bindings_sys::llama_split_mode::from)), + llama_cpp_split_modes.map(Ok) ); } diff --git a/llama-cpp-bindings/src/mtmd/eval_chunk.rs b/llama-cpp-bindings/src/mtmd/eval_chunk.rs index 9aaf8887c..d21d4ea39 100644 --- a/llama-cpp-bindings/src/mtmd/eval_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/eval_chunk.rs @@ -1,7 +1,9 @@ use std::ffi::c_char; use std::ptr; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use super::mtmd_context::MtmdContext; use super::mtmd_eval_error::MtmdEvalError; @@ -39,34 +41,34 @@ fn eval_chunk_single_status_to_result( Err(MtmdEvalError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_MTMD_CTX_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_eval_chunk_single", detail: "was given a null mtmd_ctx argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_LLAMA_CTX_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_eval_chunk_single", detail: "was given a null llama_ctx argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_CHUNK_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_eval_chunk_single", detail: "was given a null chunk argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_OUT_NEW_N_PAST_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_eval_chunk_single", detail: "was given a null out_new_n_past argument", } .into()) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_mtmd_eval_chunk_single", code: i64::from(other), } @@ -119,8 +121,10 @@ pub fn eval_chunk( mod eval_chunk_single_status_to_result_tests { use std::ptr; - use llama_cpp_ffi_status::read_and_free_cpp_string; - use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; + use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer::wrapper_allocated_error_message_pointer; use super::eval_chunk_single_status_to_result; use crate::mtmd::mtmd_eval_error::MtmdEvalError; @@ -196,7 +200,7 @@ mod eval_chunk_single_status_to_result_tests { assert_eq!( result, - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_eval_chunk_single", detail: "reported a thrown C++ exception without an error message", } @@ -210,12 +214,10 @@ mod eval_chunk_single_status_to_result_tests { assert_eq!( result, - Err(MtmdEvalError::FfiStatus( - llama_cpp_ffi_status::FfiStatusError { - operation: "llama_rs_mtmd_eval_chunk_single", - code: 255, - } - )) + Err(MtmdEvalError::FfiStatus(FfiStatusError { + operation: "llama_rs_mtmd_eval_chunk_single", + code: 255, + })) ); } @@ -230,7 +232,7 @@ mod eval_chunk_single_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_eval_chunk_single", detail: "was given a null mtmd_ctx argument", } @@ -250,7 +252,7 @@ mod eval_chunk_single_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_eval_chunk_single", detail: "was given a null llama_ctx argument", } @@ -270,7 +272,7 @@ mod eval_chunk_single_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_eval_chunk_single", detail: "was given a null chunk argument", } @@ -290,7 +292,7 @@ mod eval_chunk_single_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_eval_chunk_single", detail: "was given a null out_new_n_past argument", } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs index 56bfb6c26..fe99face6 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs @@ -5,7 +5,9 @@ use std::ptr; use std::ptr::NonNull; use std::slice; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use super::mtmd_bitmap_error::MtmdBitmapError; use super::mtmd_context::MtmdContext; @@ -37,7 +39,7 @@ unsafe fn from_file_status_to_result( match status { llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_OK => { let bitmap = NonNull::new(out_bitmap).ok_or_else(|| { - MtmdBitmapError::from(llama_cpp_ffi_status::FfiContractError { + MtmdBitmapError::from(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "success status contained a null bitmap", }) @@ -59,22 +61,22 @@ unsafe fn from_file_status_to_result( let message = unsafe { read_and_free_cpp_string(out_error, "llama_rs_mtmd_bitmap_init_from_file", "reported a thrown C++ exception without an error message") }?; Err(MtmdBitmapError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_CTX_ARG => Err(llama_cpp_ffi_status::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_CTX_ARG => Err(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "was given a null ctx argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_FNAME_ARG => Err(llama_cpp_ffi_status::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_FNAME_ARG => Err(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "was given a null fname argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_OUT_BITMAP_ARG => Err(llama_cpp_ffi_status::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_OUT_BITMAP_ARG => Err(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "was given a null out_bitmap argument", } .into()), - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_mtmd_bitmap_init_from_file", code: i64::from(other), } @@ -96,7 +98,7 @@ unsafe fn from_buf_status_to_result( match status { llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_BUF_OK => { let bitmap = NonNull::new(out_bitmap).ok_or_else(|| { - MtmdBitmapError::from(llama_cpp_ffi_status::FfiContractError { + MtmdBitmapError::from(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail: "success status contained a null bitmap", }) @@ -116,22 +118,22 @@ unsafe fn from_buf_status_to_result( let message = unsafe { read_and_free_cpp_string(out_error, "llama_rs_mtmd_bitmap_init_from_buf", "reported a thrown C++ exception without an error message") }?; Err(MtmdBitmapError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_BUF_NULL_CTX_ARG => Err(llama_cpp_ffi_status::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_BUF_NULL_CTX_ARG => Err(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail: "was given a null ctx argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_BUF_NULL_BUF_ARG => Err(llama_cpp_ffi_status::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_BUF_NULL_BUF_ARG => Err(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail: "was given a null buf argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_BUF_NULL_OUT_BITMAP_ARG => Err(llama_cpp_ffi_status::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_BUF_NULL_OUT_BITMAP_ARG => Err(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail: "was given a null out_bitmap argument", } .into()), - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_mtmd_bitmap_init_from_buf", code: i64::from(other), } @@ -150,21 +152,19 @@ fn bitmap_init_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_OK => NonNull::new(out_bitmap) .map(|bitmap| MtmdBitmap { bitmap }) .ok_or_else(|| { - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: OPERATION, detail: "success status contained a null bitmap", } .into() }), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_NULL_DATA_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { - operation: OPERATION, - detail: "was given a null data argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_NULL_DATA_ARG => Err(FfiContractError { + operation: OPERATION, + detail: "was given a null data argument", } + .into()), llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_NULL_OUT_BITMAP_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: OPERATION, detail: "was given a null out_bitmap argument", } @@ -187,7 +187,7 @@ fn bitmap_init_status_to_result( Err(MtmdBitmapError::Reported { message }) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: OPERATION, code: i64::from(other), } @@ -206,18 +206,18 @@ fn bitmap_init_from_audio_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_OK => NonNull::new(out_bitmap) .map(|bitmap| MtmdBitmap { bitmap }) .ok_or_else(|| { - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: OPERATION, detail: "success status contained a null bitmap", } .into() }), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_DATA_ARG => Err(llama_cpp_ffi_status::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_DATA_ARG => Err(FfiContractError { operation: OPERATION, detail: "was given a null data argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_OUT_BITMAP_ARG => Err(llama_cpp_ffi_status::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_OUT_BITMAP_ARG => Err(FfiContractError { operation: OPERATION, detail: "was given a null out_bitmap argument", } @@ -239,7 +239,7 @@ fn bitmap_init_from_audio_status_to_result( Err(MtmdBitmapError::Reported { message }) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: OPERATION, code: i64::from(other), } @@ -550,7 +550,8 @@ mod from_file_status_to_result_tests { use std::path::PathBuf; use std::ptr; - use llama_cpp_ffi_status::FfiStatusError; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; use super::from_file_status_to_result; use crate::mtmd::mtmd_bitmap_error::MtmdBitmapError; @@ -568,7 +569,7 @@ mod from_file_status_to_result_tests { assert_eq!( result.unwrap_err(), - MtmdBitmapError::FfiContract(llama_cpp_ffi_status::FfiContractError { + MtmdBitmapError::FfiContract(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "success status contained a null bitmap", }) @@ -621,7 +622,7 @@ mod from_file_status_to_result_tests { assert_eq!( result.unwrap_err(), - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "reported a thrown C++ exception without an error message", } @@ -643,7 +644,7 @@ mod from_file_status_to_result_tests { assert_eq!( result.unwrap_err(), - MtmdBitmapError::FfiContract(llama_cpp_ffi_status::FfiContractError { + MtmdBitmapError::FfiContract(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "was given a null ctx argument", }) @@ -663,7 +664,7 @@ mod from_file_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "was given a null fname argument", } @@ -685,7 +686,7 @@ mod from_file_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "was given a null out_bitmap argument", } @@ -749,8 +750,10 @@ mod from_file_status_to_result_tests { mod from_buf_status_to_result_tests { use std::ptr; - use llama_cpp_ffi_status::read_and_free_cpp_string; - use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; + use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer::wrapper_allocated_error_message_pointer; use super::from_buf_status_to_result; use crate::mtmd::mtmd_bitmap_error::MtmdBitmapError; @@ -789,7 +792,7 @@ mod from_buf_status_to_result_tests { assert_eq!( result.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail: "success status contained a null bitmap", } @@ -850,7 +853,7 @@ mod from_buf_status_to_result_tests { assert_eq!( result.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail: "reported a thrown C++ exception without an error message", } @@ -909,7 +912,7 @@ mod from_buf_status_to_result_tests { assert_eq!( result.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail, } @@ -929,7 +932,7 @@ mod from_buf_status_to_result_tests { assert_eq!( result.err(), Some( - llama_cpp_ffi_status::FfiStatusError { + FfiStatusError { operation: "llama_rs_mtmd_bitmap_init_from_buf", code: i64::from(unrecognized), } @@ -943,12 +946,15 @@ mod from_buf_status_to_result_tests { mod bitmap_init_status_to_result_tests { use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::bitmap_init_from_audio_status_to_result; use super::bitmap_init_status_to_result; use crate::mtmd::mtmd_bitmap_error::MtmdBitmapError; fn contract_violation(operation: &'static str, detail: &'static str) -> MtmdBitmapError { - llama_cpp_ffi_status::FfiContractError { operation, detail }.into() + FfiContractError { operation, detail }.into() } #[test] @@ -987,12 +993,10 @@ mod bitmap_init_status_to_result_tests { operation, "reported a thrown C++ exception without an error message" )), - Some(MtmdBitmapError::FfiStatus( - llama_cpp_ffi_status::FfiStatusError { - operation, - code: 255, - } - )), + Some(MtmdBitmapError::FfiStatus(FfiStatusError { + operation, + code: 255, + })), ] ); } @@ -1034,12 +1038,10 @@ mod bitmap_init_status_to_result_tests { operation, "reported a thrown C++ exception without an error message" )), - Some(MtmdBitmapError::FfiStatus( - llama_cpp_ffi_status::FfiStatusError { - operation, - code: 255, - } - )), + Some(MtmdBitmapError::FfiStatus(FfiStatusError { + operation, + code: 255, + })), ] ); } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_bitmap_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_bitmap_error.rs index dcb70d98e..5429c14b9 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_bitmap_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_bitmap_error.rs @@ -1,11 +1,14 @@ use std::path::PathBuf; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdBitmapError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("Failed to create CString from bitmap-source path: {0}")] CStringError(#[from] std::ffi::NulError), #[error("Bitmap-source path is not valid UTF-8: {0:?}")] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_context.rs b/llama-cpp-bindings/src/mtmd/mtmd_context.rs index 227f611aa..7685909a5 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_context.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_context.rs @@ -6,7 +6,9 @@ use std::ptr::NonNull; use enumflags2::BitFlags; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use super::mtmd_bitmap::MtmdBitmap; use super::mtmd_capability::MtmdCapability; @@ -81,28 +83,28 @@ fn map_tokenize_status( Err(MtmdTokenizeError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_BITMAPS_ARG_WHEN_NUM_BITMAPS_NONZERO => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "nonzero bitmap count was observed with a null bitmap array", } .into()) } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_CTX_ARG => Err(llama_cpp_ffi_status::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_CTX_ARG => Err(FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "was given a null ctx argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_OUTPUT_ARG => Err(llama_cpp_ffi_status::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_OUTPUT_ARG => Err(FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "was given a null output argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_TEXT_ARG => Err(llama_cpp_ffi_status::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_TEXT_ARG => Err(FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "was given a null text argument", } .into()), - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_mtmd_tokenize", code: i64::from(other), } @@ -118,7 +120,7 @@ fn map_tokenize_from_parts_status( match status { llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_OK => Ok(()), llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_LLAMA_CPP_REPORTED_MALFORMED_PART => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "llama.cpp reported a malformed part list: a part holding both or neither of text and bitmap, a text part with a null text pointer, or bitmaps that do not line up with their parts", } @@ -149,34 +151,34 @@ fn map_tokenize_from_parts_status( Err(MtmdTokenizeError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_NULL_CTX_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "was given a null ctx argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_NULL_OUTPUT_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "was given a null output argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_NULL_PARTS_ARG_WHEN_NUM_PARTS_NONZERO => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "nonzero part count was observed with a null part array", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_NULL_PART_ELEMENT => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "was given a null element in the parts argument", } .into()) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_mtmd_tokenize_from_parts", code: i64::from(other), } @@ -212,21 +214,19 @@ fn map_encode_chunk_status( }?; Err(MtmdEncodeError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_ENCODE_CHUNK_NULL_CTX_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_mtmd_encode_chunk", - detail: "was given a null ctx argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_MTMD_ENCODE_CHUNK_NULL_CTX_ARG => Err(FfiContractError { + operation: "llama_rs_mtmd_encode_chunk", + detail: "was given a null ctx argument", } + .into()), llama_cpp_bindings_sys::LLAMA_RS_MTMD_ENCODE_CHUNK_NULL_CHUNK_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_encode_chunk", detail: "was given a null chunk argument", } .into()) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_mtmd_encode_chunk", code: i64::from(other), } @@ -243,7 +243,7 @@ fn map_init_from_file_status( match status { llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_OK => { let context = NonNull::new(out_ctx).ok_or_else(|| { - MtmdInitError::from(llama_cpp_ffi_status::FfiContractError { + MtmdInitError::from(FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "success status contained a null multimodal context", }) @@ -272,27 +272,27 @@ fn map_init_from_file_status( Err(MtmdInitError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_NULL_MMPROJ_PATH_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "was given a null mmproj_path argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_NULL_TEXT_MODEL_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "was given a null text_model argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_NULL_OUT_CTX_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "was given a null out_ctx argument", } .into()) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_mtmd_init_from_file", code: i64::from(other), } @@ -478,8 +478,10 @@ impl Drop for MtmdContext { mod map_tokenize_from_parts_status_tests { use std::ptr; - use llama_cpp_ffi_status::read_and_free_cpp_string; - use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; + use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer::wrapper_allocated_error_message_pointer; use super::map_tokenize_from_parts_status; use crate::mtmd::mtmd_tokenize_error::MtmdTokenizeError; @@ -577,7 +579,7 @@ mod map_tokenize_from_parts_status_tests { ) .err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "llama.cpp reported a malformed part list: a part holding both or neither of text and bitmap, a text part with a null text pointer, or bitmaps that do not line up with their parts", } @@ -596,7 +598,7 @@ mod map_tokenize_from_parts_status_tests { ) .err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "was given a null element in the parts argument", } @@ -615,7 +617,7 @@ mod map_tokenize_from_parts_status_tests { ) .err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "reported a thrown C++ exception without an error message", } @@ -634,7 +636,7 @@ mod map_tokenize_from_parts_status_tests { ) .err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "was given a null ctx argument", } @@ -653,7 +655,7 @@ mod map_tokenize_from_parts_status_tests { ) .err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "was given a null output argument", } @@ -672,7 +674,7 @@ mod map_tokenize_from_parts_status_tests { ) .err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "nonzero part count was observed with a null part array", } @@ -681,12 +683,10 @@ mod map_tokenize_from_parts_status_tests { ); assert_eq!( map_tokenize_from_parts_status(255, 0, ptr::null_mut()).err(), - Some(MtmdTokenizeError::FfiStatus( - llama_cpp_ffi_status::FfiStatusError { - operation: "llama_rs_mtmd_tokenize_from_parts", - code: 255, - } - )) + Some(MtmdTokenizeError::FfiStatus(FfiStatusError { + operation: "llama_rs_mtmd_tokenize_from_parts", + code: 255, + })) ); } } @@ -695,6 +695,9 @@ mod map_tokenize_from_parts_status_tests { mod map_tokenize_status_tests { use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::map_tokenize_status; use crate::mtmd::mtmd_tokenize_error::MtmdTokenizeError; @@ -769,7 +772,7 @@ mod map_tokenize_status_tests { assert_eq!( result, - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "reported a thrown C++ exception without an error message", } @@ -787,12 +790,10 @@ mod map_tokenize_status_tests { assert_eq!( result, - Err(MtmdTokenizeError::FfiContract( - llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_mtmd_tokenize", - detail: "nonzero bitmap count was observed with a null bitmap array", - } - )) + Err(MtmdTokenizeError::FfiContract(FfiContractError { + operation: "llama_rs_mtmd_tokenize", + detail: "nonzero bitmap count was observed with a null bitmap array", + })) ); } @@ -802,12 +803,10 @@ mod map_tokenize_status_tests { assert_eq!( result, - Err(MtmdTokenizeError::FfiStatus( - llama_cpp_ffi_status::FfiStatusError { - operation: "llama_rs_mtmd_tokenize", - code: 255, - } - )) + Err(MtmdTokenizeError::FfiStatus(FfiStatusError { + operation: "llama_rs_mtmd_tokenize", + code: 255, + })) ); } @@ -821,7 +820,7 @@ mod map_tokenize_status_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "was given a null ctx argument", } @@ -840,7 +839,7 @@ mod map_tokenize_status_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "was given a null output argument", } @@ -859,7 +858,7 @@ mod map_tokenize_status_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "was given a null text argument", } @@ -900,6 +899,9 @@ mod map_tokenize_status_tests { mod map_encode_chunk_status_tests { use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::map_encode_chunk_status; use crate::mtmd::mtmd_encode_error::MtmdEncodeError; @@ -946,7 +948,7 @@ mod map_encode_chunk_status_tests { assert_eq!( result, - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_encode_chunk", detail: "reported a thrown C++ exception without an error message", } @@ -960,12 +962,10 @@ mod map_encode_chunk_status_tests { assert_eq!( result, - Err(MtmdEncodeError::FfiStatus( - llama_cpp_ffi_status::FfiStatusError { - operation: "llama_rs_mtmd_encode_chunk", - code: 255, - } - )) + Err(MtmdEncodeError::FfiStatus(FfiStatusError { + operation: "llama_rs_mtmd_encode_chunk", + code: 255, + })) ); } @@ -979,7 +979,7 @@ mod map_encode_chunk_status_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_encode_chunk", detail: "was given a null ctx argument", } @@ -998,7 +998,7 @@ mod map_encode_chunk_status_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_encode_chunk", detail: "was given a null chunk argument", } @@ -1039,6 +1039,9 @@ mod map_encode_chunk_status_tests { mod map_init_from_file_status_tests { use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::map_init_from_file_status; use crate::mtmd::mtmd_init_error::MtmdInitError; @@ -1053,7 +1056,7 @@ mod map_init_from_file_status_tests { assert_eq!( result.unwrap_err(), - MtmdInitError::FfiContract(llama_cpp_ffi_status::FfiContractError { + MtmdInitError::FfiContract(FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "success status contained a null multimodal context", }) @@ -1083,7 +1086,7 @@ mod map_init_from_file_status_tests { assert_eq!( result.unwrap_err(), - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "reported a thrown C++ exception without an error message", } @@ -1098,7 +1101,7 @@ mod map_init_from_file_status_tests { assert_eq!( result.unwrap_err(), - MtmdInitError::FfiStatus(llama_cpp_ffi_status::FfiStatusError { + MtmdInitError::FfiStatus(FfiStatusError { operation: "llama_rs_mtmd_init_from_file", code: 255, }) @@ -1116,7 +1119,7 @@ mod map_init_from_file_status_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "was given a null mmproj_path argument", } @@ -1136,7 +1139,7 @@ mod map_init_from_file_status_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "was given a null text_model argument", } @@ -1156,7 +1159,7 @@ mod map_init_from_file_status_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "was given a null out_ctx argument", } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_encode_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_encode_error.rs index 7f08ccaf7..c2f3efc3c 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_encode_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_encode_error.rs @@ -1,9 +1,12 @@ +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdEncodeError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("multimodal chunk encoding failed with code: {code}")] EncodingFailed { code: i32 }, #[error("not enough memory")] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs index d77f03e35..9f3ac9e07 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs @@ -1,3 +1,6 @@ +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; use crate::mtmd::mtmd_input_chunk_type_error::MtmdInputChunkTypeError; use crate::mtmd::non_causal_chunk_micro_batch_mismatch::NonCausalChunkMicroBatchMismatch; @@ -5,9 +8,9 @@ use crate::mtmd::non_causal_chunk_micro_batch_mismatch::NonCausalChunkMicroBatch #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdEvalError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("batch size {requested} exceeds context batch size {context_max}")] BatchSizeExceedsContextLimit { requested: i32, context_max: u32 }, #[error("batch size {requested} must be positive")] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_init_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_init_error.rs index 03190edb5..d4b0ce4a4 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_init_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_init_error.rs @@ -1,13 +1,16 @@ use std::path::PathBuf; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::mtmd_context_params_error::MtmdContextParamsError; #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdInitError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error(transparent)] ContextParams(#[from] MtmdContextParamsError), #[error("Failed to create CString from mmproj path: {0}")] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs index a67f2e8b0..d5685bb60 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs @@ -4,7 +4,9 @@ use std::ptr; use std::ptr::NonNull; use std::slice; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use super::mtmd_context::MtmdContext; use super::mtmd_eval_error::MtmdEvalError; @@ -64,21 +66,21 @@ fn input_chunk_copy_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_OK => NonNull::new(out_chunk) .map(|chunk| MtmdInputChunk { chunk, owned: true }) .ok_or_else(|| { - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: OPERATION, detail: "success status contained a null chunk", } .into() }), llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_NULL_CHUNK_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: OPERATION, detail: "was given a null chunk argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_NULL_OUT_CHUNK_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: OPERATION, detail: "was given a null out_chunk argument", } @@ -101,7 +103,7 @@ fn input_chunk_copy_status_to_result( Err(MtmdInputChunkError::Reported { message }) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: OPERATION, code: i64::from(other), } @@ -274,13 +276,16 @@ mod unit_tests { mod input_chunk_copy_status_to_result_tests { use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::input_chunk_copy_status_to_result; use crate::mtmd::mtmd_input_chunk_error::MtmdInputChunkError; const OPERATION: &str = "llama_rs_mtmd_input_chunk_copy"; fn contract_violation(detail: &'static str) -> MtmdInputChunkError { - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: OPERATION, detail, } @@ -312,12 +317,10 @@ mod input_chunk_copy_status_to_result_tests { Some(contract_violation( "reported a thrown C++ exception without an error message" )), - Some(MtmdInputChunkError::FfiStatus( - llama_cpp_ffi_status::FfiStatusError { - operation: OPERATION, - code: 255, - } - )), + Some(MtmdInputChunkError::FfiStatus(FfiStatusError { + operation: OPERATION, + code: 255, + })), ] ); } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_error.rs index b1607a042..928d551a0 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_error.rs @@ -1,9 +1,12 @@ +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdInputChunkError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs index 1e1c92e70..a79a82bcd 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs @@ -2,7 +2,9 @@ use std::ffi::c_char; use std::ptr; use std::ptr::NonNull; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use super::eval_chunk::eval_chunk; use super::mtmd_context::MtmdContext; @@ -24,14 +26,14 @@ fn input_chunks_init_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_OK => NonNull::new(out_chunks) .map(|chunks| MtmdInputChunks { chunks }) .ok_or_else(|| { - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: OPERATION, detail: "success status contained null chunks", } .into() }), llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_NULL_OUT_CHUNKS_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: OPERATION, detail: "was given a null out_chunks argument", } @@ -54,7 +56,7 @@ fn input_chunks_init_status_to_result( Err(MtmdInputChunksError::Reported { message }) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: OPERATION, code: i64::from(other), } @@ -196,7 +198,7 @@ impl MtmdInputChunks { (0..self.len()) .map(|index| { self.get(index).ok_or_else(|| { - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "mtmd_input_chunks_get", detail: "returned a null chunk within the chunk count", } @@ -238,13 +240,16 @@ mod tests { mod input_chunks_init_status_to_result_tests { use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::input_chunks_init_status_to_result; use crate::mtmd::mtmd_input_chunks_error::MtmdInputChunksError; const OPERATION: &str = "llama_rs_mtmd_input_chunks_init"; fn contract_violation(detail: &'static str) -> MtmdInputChunksError { - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: OPERATION, detail, } @@ -274,12 +279,10 @@ mod input_chunks_init_status_to_result_tests { Some(contract_violation( "reported a thrown C++ exception without an error message" )), - Some(MtmdInputChunksError::FfiStatus( - llama_cpp_ffi_status::FfiStatusError { - operation: OPERATION, - code: 255, - } - )), + Some(MtmdInputChunksError::FfiStatus(FfiStatusError { + operation: OPERATION, + code: 255, + })), ] ); } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks_error.rs index 6b2e68385..acd451eea 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks_error.rs @@ -1,9 +1,12 @@ +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdInputChunksError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs index 456b9e0b0..21340f270 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs @@ -3,7 +3,9 @@ use std::ffi::c_char; use std::num::NonZeroU32; use std::ptr; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use super::mtmd_context_params::MtmdContextParams; use super::mtmd_image_token_limit::MtmdImageTokenLimit; @@ -59,7 +61,7 @@ fn get_memory_usage_status_to_result( Err(MtmdMemoryUsageError::Reported { message }) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: OPERATION, code: i64::from(other), } @@ -68,7 +70,7 @@ fn get_memory_usage_status_to_result( } const fn contract_violation(detail: &'static str) -> MtmdMemoryUsageError { - MtmdMemoryUsageError::FfiContract(llama_cpp_ffi_status::FfiContractError { + MtmdMemoryUsageError::FfiContract(FfiContractError { operation: OPERATION, detail, }) @@ -179,8 +181,10 @@ mod tests { use std::num::NonZeroU32; use std::ptr; - use llama_cpp_fault_injection::ArmedFault; - use llama_cpp_fault_injection::injected_fault_message; + use llama_cpp_fault_injection::armed_fault::ArmedFault; + use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; use super::get_memory_usage_status_to_result; use crate::mtmd::mtmd_context_params::MtmdContextParams; @@ -218,7 +222,7 @@ mod tests { } fn contract_violation(detail: &'static str) -> MtmdMemoryUsageError { - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_get_memory_usage", detail, } @@ -426,12 +430,10 @@ mod tests { fn an_unknown_status_is_preserved() { assert_eq!( get_memory_usage_status_to_result(255, ptr::null_mut()), - Err(MtmdMemoryUsageError::FfiStatus( - llama_cpp_ffi_status::FfiStatusError { - operation: "llama_rs_mtmd_get_memory_usage", - code: 255, - } - )) + Err(MtmdMemoryUsageError::FfiStatus(FfiStatusError { + operation: "llama_rs_mtmd_get_memory_usage", + code: 255, + })) ); } } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage_error.rs index 0d9417d19..e5983ecc4 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage_error.rs @@ -1,13 +1,16 @@ use std::ffi::CString; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::mtmd_context_params_error::MtmdContextParamsError; #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdMemoryUsageError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error(transparent)] ContextParams(#[from] MtmdContextParamsError), #[error("the memory usage of mmproj {path:?} could not be measured, because it does not load")] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_tokenize_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_tokenize_error.rs index d50f828cc..b67026c0c 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_tokenize_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_tokenize_error.rs @@ -1,11 +1,14 @@ +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use crate::mtmd::mtmd_input_chunks_error::MtmdInputChunksError; #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdTokenizeError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("Failed to create CString from input text: {0}")] CStringError(#[from] std::ffi::NulError), #[error("{0}")] diff --git a/llama-cpp-bindings/src/parsed_chat.rs b/llama-cpp-bindings/src/parsed_chat.rs index ff2af3121..897c09f24 100644 --- a/llama-cpp-bindings/src/parsed_chat.rs +++ b/llama-cpp-bindings/src/parsed_chat.rs @@ -1,12 +1,12 @@ use std::ffi::c_char; use std::ptr::NonNull; -use llama_cpp_bindings_types::ModelProvidedToolCallId; -use llama_cpp_bindings_types::ModelProvidedToolCallIdError; -use llama_cpp_bindings_types::ParsedChatMessage; -use llama_cpp_bindings_types::ParsedToolCall; -use llama_cpp_bindings_types::ParsedToolCallId; -use llama_cpp_bindings_types::ToolCallArguments; +use llama_cpp_bindings_types::model_provided_tool_call_id::ModelProvidedToolCallId; +use llama_cpp_bindings_types::model_provided_tool_call_id_error::ModelProvidedToolCallIdError; +use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; +use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; +use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; +use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; use crate::cpp_text_view::cpp_text_view; use crate::error::parse_chat_message_error::ParseChatMessageError; @@ -93,8 +93,8 @@ pub fn take_parsed_chat_message( #[cfg(test)] mod tests { - use llama_cpp_bindings_types::ModelProvidedToolCallId; - use llama_cpp_bindings_types::ParsedToolCallId; + use llama_cpp_bindings_types::model_provided_tool_call_id::ModelProvidedToolCallId; + use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; use super::tool_call_id_reported_by_llama_cpp; diff --git a/llama-cpp-bindings/src/sampled_token_classifier.rs b/llama-cpp-bindings/src/sampled_token_classifier.rs index 6cf21432f..e820f6890 100644 --- a/llama-cpp-bindings/src/sampled_token_classifier.rs +++ b/llama-cpp-bindings/src/sampled_token_classifier.rs @@ -1,7 +1,7 @@ use llama_cpp_bindings_sys::llama_pos; use llama_cpp_bindings_sys::llama_seq_id; -use llama_cpp_bindings_types::TokenUsage; -use llama_cpp_bindings_types::TokenUsageError; +use llama_cpp_bindings_types::token_usage::TokenUsage; +use llama_cpp_bindings_types::token_usage_error::TokenUsageError; use crate::bare_json_tool_call_probe::BareJsonToolCallProbe; use crate::bare_json_tool_calls::BareJsonToolCalls; diff --git a/llama-cpp-bindings/src/sampling.rs b/llama-cpp-bindings/src/sampling.rs index 35bbbd660..07f18ef6f 100644 --- a/llama-cpp-bindings/src/sampling.rs +++ b/llama-cpp-bindings/src/sampling.rs @@ -6,9 +6,11 @@ use std::mem; use std::ptr; use std::ptr::NonNull; -use llama_cpp_error_recorder::ErrorScope; -use llama_cpp_error_recorder::RecordedError; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use llama_cpp_error_recorder::error_scope::ErrorScope; +use llama_cpp_error_recorder::recorded_error::RecordedError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use llama_cpp_gbnf::gbnf_validation_error::GbnfValidationError; use llama_cpp_gbnf::validate_gbnf::validate_gbnf; @@ -48,21 +50,19 @@ fn check_sampler_accept_status( }?; Err(SamplerAcceptError::GrammarStateCorrupted { message }) } - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_NULL_SAMPLER_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_sampler_accept", - detail: "was given a null sampler argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_NULL_SAMPLER_ARG => Err(FfiContractError { + operation: "llama_rs_sampler_accept", + detail: "was given a null sampler argument", } + .into()), llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_NULL_OUT_ERROR_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_accept", detail: "was given a null out_error argument", } .into()) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_sampler_accept", code: i64::from(other), } @@ -96,20 +96,20 @@ fn sampler_apply_status_to_result( Err(SamplerApplyError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_DATA_ARRAY_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_apply", detail: "was given a null data_array argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_OUT_ERROR_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_apply", detail: "was given a null out_error argument", } .into()) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_sampler_apply", code: i64::from(other), } @@ -140,35 +140,31 @@ fn sampler_sample_status_to_result( }?; Err(SampleError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_SAMPLER_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_sampler_sample", - detail: "was given a null sampler argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_SAMPLER_ARG => Err(FfiContractError { + operation: "llama_rs_sampler_sample", + detail: "was given a null sampler argument", } - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_CTX_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_sampler_sample", - detail: "was given a null ctx argument", - } - .into()) + .into()), + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_CTX_ARG => Err(FfiContractError { + operation: "llama_rs_sampler_sample", + detail: "was given a null ctx argument", } + .into()), llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_OUT_TOKEN_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_sample", detail: "was given a null out_token argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_OUT_ERROR_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_sample", detail: "was given a null out_error argument", } .into()) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_sampler_sample", code: i64::from(other), } @@ -183,7 +179,7 @@ fn sampler_init_grammar_status_to_result( ) -> Result { match status { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_OK => { - LlamaSampler::from_raw(sampler, "grammar").map_err(Into::into) + unsafe { LlamaSampler::from_raw(sampler, "grammar") }.map_err(Into::into) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LLAMA_CPP_RETURNED_NULL => { Err(GrammarError::GrammarMalformed) @@ -205,20 +201,20 @@ fn sampler_init_grammar_status_to_result( Err(GrammarError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_NULL_OUT_SAMPLER_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_init_grammar", detail: "was given a null out_sampler argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_NULL_OUT_ERROR_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_init_grammar", detail: "was given a null out_error argument", } .into()) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_sampler_init_grammar", code: i64::from(other), } @@ -233,7 +229,7 @@ fn sampler_init_grammar_lazy_patterns_status_to_result( ) -> Result { match status { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_OK => { - LlamaSampler::from_raw(sampler, "lazy grammar").map_err(Into::into) + unsafe { LlamaSampler::from_raw(sampler, "lazy grammar") }.map_err(Into::into) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_LLAMA_CPP_RETURNED_NULL => { Err(GrammarError::LazyGrammarMalformed) @@ -252,17 +248,17 @@ fn sampler_init_grammar_lazy_patterns_status_to_result( let message = unsafe { read_and_free_cpp_string(error_ptr, "llama_rs_sampler_init_grammar_lazy_patterns", "reported a thrown C++ exception without an error message") }?; Err(GrammarError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_NULL_OUT_SAMPLER_ARG => Err(llama_cpp_ffi_status::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_NULL_OUT_SAMPLER_ARG => Err(FfiContractError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", detail: "was given a null out_sampler argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_NULL_OUT_ERROR_ARG => Err(llama_cpp_ffi_status::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_NULL_OUT_ERROR_ARG => Err(FfiContractError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", detail: "was given a null out_error argument", } .into()), - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", code: i64::from(other), } @@ -277,9 +273,9 @@ fn sampler_init_status_to_result( sampler_name: &'static str, ) -> Result { match status { - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_OK => { + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_OK => unsafe { LlamaSampler::from_raw(sampler, sampler_name) - } + }, llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_LLAMA_CPP_RETURNED_NULL => { Err(SamplingError::SamplerUnavailable { sampler: sampler_name, @@ -307,20 +303,18 @@ fn sampler_init_status_to_result( }) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_NULL_OUT_SAMPLER_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_init", detail: "was given a null out_sampler argument", } .into()) } - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_NULL_OUT_ERROR_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_sampler_init", - detail: "was given a null out_error argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_NULL_OUT_ERROR_ARG => Err(FfiContractError { + operation: "llama_rs_sampler_init", + detail: "was given a null out_error argument", } - other => Err(llama_cpp_ffi_status::FfiStatusError { + .into()), + other => Err(FfiStatusError { operation: "llama_rs_sampler_init", code: i64::from(other), } @@ -354,27 +348,27 @@ fn sampler_chain_add_status_to_result( }) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_NULL_CHAIN_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "was given a null chain argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_NULL_SAMPLER_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "was given a null sampler argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_NULL_OUT_ERROR_ARG => { - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "was given a null out_error argument", } .into()) } - other => Err(llama_cpp_ffi_status::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_sampler_chain_add", code: i64::from(other), } @@ -417,7 +411,17 @@ impl Debug for LlamaSampler { } impl LlamaSampler { - pub(crate) fn from_raw( + /// Takes ownership of a sampler llama.cpp initialized, and frees it on drop. + /// + /// # Safety + /// + /// `sampler` must be null or a sampler llama.cpp initialized that no other + /// [`LlamaSampler`] owns. + /// + /// # Errors + /// + /// Returns [`SamplingError::SamplerUnavailable`] when `sampler` is null. + pub unsafe fn from_raw( sampler: *mut llama_cpp_bindings_sys::llama_sampler, sampler_name: &'static str, ) -> Result { @@ -978,8 +982,8 @@ mod tests { use std::ptr; use std::slice; - use llama_cpp_error_recorder::RecordedError; - use llama_cpp_error_recorder::record; + use llama_cpp_error_recorder::record::record; + use llama_cpp_error_recorder::recorded_error::RecordedError; use super::LlamaSampler; use crate::error::grammar_error::GrammarError; @@ -1038,7 +1042,7 @@ mod tests { ) }; - LlamaSampler::from_raw(sampler, "failing excluding sampler") + unsafe { LlamaSampler::from_raw(sampler, "failing excluding sampler") } .expect("the failing excluding sampler must initialize") } @@ -1062,7 +1066,7 @@ mod tests { #[test] fn null_native_sampler_is_initialization_error() { - let result = LlamaSampler::from_raw(ptr::null_mut(), "test"); + let result = unsafe { LlamaSampler::from_raw(ptr::null_mut(), "test") }; assert_eq!( result.unwrap_err(), @@ -1187,7 +1191,7 @@ mod tests { } #[test] - fn accept_succeeds() { + fn a_penalties_chain_accepts_a_token() { let mut sampler = LlamaSampler::chain([penalties_sampler(), greedy_sampler()]) .expect("sampler chain must initialize"); @@ -1247,7 +1251,7 @@ mod tests { } #[test] - fn debug_formatting() { + fn debug_hides_the_native_sampler_pointer() { let sampler = greedy_sampler(); let debug_output = format!("{sampler:?}"); assert_eq!(debug_output, "LlamaSampler { .. }"); @@ -1287,7 +1291,7 @@ mod checked_usize_as_i32_sampling_tests { #[cfg(test)] mod grammar_callback_error_to_result_tests { - use llama_cpp_error_recorder::RecordedError; + use llama_cpp_error_recorder::recorded_error::RecordedError; use super::grammar_callback_error_to_result; use crate::error::sample_error::SampleError; @@ -1313,7 +1317,7 @@ mod grammar_callback_error_to_result_tests { #[cfg(test)] mod grammar_callback_error_to_accept_result_tests { - use llama_cpp_error_recorder::RecordedError; + use llama_cpp_error_recorder::recorded_error::RecordedError; use super::grammar_callback_error_to_accept_result; use crate::error::sampler_accept_error::SamplerAcceptError; @@ -1342,14 +1346,14 @@ mod grammar_callback_error_to_accept_result_tests { mod check_sampler_accept_status_tests { use std::ptr; - use llama_cpp_ffi_status::FfiContractError; - use llama_cpp_ffi_status::FfiStatusError; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; use super::check_sampler_accept_status; use crate::error::sampler_accept_error::SamplerAcceptError; #[test] - fn check_sampler_accept_status_ok() { + fn check_sampler_accept_status_maps_ok_to_success() { let result = check_sampler_accept_status( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_OK, ptr::null_mut(), @@ -1462,7 +1466,8 @@ mod check_sampler_accept_status_tests { mod sampler_sample_status_to_result_tests { use std::ptr; - use llama_cpp_ffi_status::FfiStatusError; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; use super::sampler_sample_status_to_result; use crate::error::sample_error::SampleError; @@ -1488,7 +1493,7 @@ mod sampler_sample_status_to_result_tests { assert_eq!( result.unwrap_err(), - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_sample", detail: "reported a thrown C++ exception without an error message", } @@ -1503,12 +1508,10 @@ mod sampler_sample_status_to_result_tests { assert_eq!( result, - Err(SampleError::FfiContract( - llama_cpp_ffi_status::FfiContractError { - operation: "llama_rs_sampler_sample", - detail: "was given a null ctx argument", - } - )) + Err(SampleError::FfiContract(FfiContractError { + operation: "llama_rs_sampler_sample", + detail: "was given a null ctx argument", + })) ); } @@ -1522,7 +1525,7 @@ mod sampler_sample_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_sample", detail: "was given a null sampler argument", } @@ -1541,7 +1544,7 @@ mod sampler_sample_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_sample", detail: "was given a null out_token argument", } @@ -1560,7 +1563,7 @@ mod sampler_sample_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_sample", detail: "was given a null out_error argument", } @@ -1619,6 +1622,9 @@ mod sampler_sample_status_to_result_tests { mod sampler_init_status_to_result_tests { use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::sampler_init_status_to_result; use crate::error::sampling_error::SamplingError; @@ -1699,7 +1705,7 @@ mod sampler_init_status_to_result_tests { ) .err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_init", detail: "was given a null out_sampler argument", } @@ -1719,7 +1725,7 @@ mod sampler_init_status_to_result_tests { ) .err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_init", detail: "was given a null out_error argument", } @@ -1739,7 +1745,7 @@ mod sampler_init_status_to_result_tests { ) .err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_init", detail: "reported a thrown C++ exception without an error message", } @@ -1759,7 +1765,7 @@ mod sampler_init_status_to_result_tests { ) .err(), Some( - llama_cpp_ffi_status::FfiStatusError { + FfiStatusError { operation: "llama_rs_sampler_init", code: i64::from( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION @@ -1776,6 +1782,9 @@ mod sampler_init_status_to_result_tests { mod sampler_chain_add_status_to_result_tests { use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::sampler_chain_add_status_to_result; use crate::error::sampling_error::SamplingError; @@ -1825,7 +1834,7 @@ mod sampler_chain_add_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_NULL_CHAIN_ARG, ptr::null_mut(), ), - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "was given a null chain argument", } @@ -1840,7 +1849,7 @@ mod sampler_chain_add_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_NULL_SAMPLER_ARG, ptr::null_mut(), ), - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "was given a null sampler argument", } @@ -1855,7 +1864,7 @@ mod sampler_chain_add_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_NULL_OUT_ERROR_ARG, ptr::null_mut(), ), - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "was given a null out_error argument", } @@ -1870,7 +1879,7 @@ mod sampler_chain_add_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), ), - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "reported a thrown C++ exception without an error message", } @@ -1886,7 +1895,7 @@ mod sampler_chain_add_status_to_result_tests { + 1, ptr::null_mut(), ), - Err(llama_cpp_ffi_status::FfiStatusError { + Err(FfiStatusError { operation: "llama_rs_sampler_chain_add", code: i64::from( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION @@ -1902,7 +1911,8 @@ mod sampler_chain_add_status_to_result_tests { mod sampler_init_grammar_status_to_result_tests { use std::ptr; - use llama_cpp_ffi_status::FfiStatusError; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; use super::sampler_init_grammar_status_to_result; use crate::error::grammar_error::GrammarError; @@ -1939,7 +1949,7 @@ mod sampler_init_grammar_status_to_result_tests { assert_eq!( result.unwrap_err(), - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_init_grammar", detail: "reported a thrown C++ exception without an error message", } @@ -1955,7 +1965,7 @@ mod sampler_init_grammar_status_to_result_tests { assert_eq!( result.unwrap_err(), - GrammarError::FfiContract(llama_cpp_ffi_status::FfiContractError { + GrammarError::FfiContract(FfiContractError { operation: "llama_rs_sampler_init_grammar", detail: "was given a null out_sampler argument", }) @@ -1972,7 +1982,7 @@ mod sampler_init_grammar_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_init_grammar", detail: "was given a null out_error argument", } @@ -2034,7 +2044,8 @@ mod sampler_init_grammar_status_to_result_tests { mod sampler_init_grammar_lazy_patterns_status_to_result_tests { use std::ptr; - use llama_cpp_ffi_status::FfiStatusError; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; use super::sampler_init_grammar_lazy_patterns_status_to_result; use crate::error::grammar_error::GrammarError; @@ -2071,7 +2082,7 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { assert_eq!( result.unwrap_err(), - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", detail: "reported a thrown C++ exception without an error message", } @@ -2089,7 +2100,7 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { ) .err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", detail: "reported a thrown C++ exception without an error message", } @@ -2109,7 +2120,7 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { assert_eq!( result.unwrap_err(), - GrammarError::FfiContract(llama_cpp_ffi_status::FfiContractError { + GrammarError::FfiContract(FfiContractError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", detail: "was given a null out_sampler argument", }) @@ -2126,7 +2137,7 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", detail: "was given a null out_error argument", } @@ -2184,8 +2195,9 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { mod sampler_application_tests { use std::ptr; - use llama_cpp_fault_injection::ArmedFault; - use llama_cpp_fault_injection::injected_fault_message; + use llama_cpp_fault_injection::armed_fault::ArmedFault; + use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use super::LlamaSampler; use crate::error::sample_error::SampleError; @@ -2244,7 +2256,7 @@ mod sampler_application_tests { assert_eq!( overreporting_sampler.apply(&mut excluded_candidates()), Err(SampleError::SamplerApply(SamplerApplyError::FfiContract( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "modify_as_c_llama_token_data_array", detail: "the llama.cpp sampler grew the token data array beyond its capacity", } @@ -2303,6 +2315,9 @@ mod sampler_application_tests { mod sampler_apply_status_to_result_tests { use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::sampler_apply_status_to_result; use crate::error::sampler_apply_error::SamplerApplyError; @@ -2324,7 +2339,7 @@ mod sampler_apply_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), ), - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_apply", detail: "reported a thrown C++ exception without an error message", } @@ -2338,12 +2353,10 @@ mod sampler_apply_status_to_result_tests { assert_eq!( result, - Err(SamplerApplyError::FfiStatus( - llama_cpp_ffi_status::FfiStatusError { - operation: "llama_rs_sampler_apply", - code: 255, - } - )) + Err(SamplerApplyError::FfiStatus(FfiStatusError { + operation: "llama_rs_sampler_apply", + code: 255, + })) ); } @@ -2365,7 +2378,7 @@ mod sampler_apply_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_apply", detail: "was given a null data_array argument", } @@ -2383,7 +2396,7 @@ mod sampler_apply_status_to_result_tests { assert_eq!( outcome.err(), Some( - llama_cpp_ffi_status::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_apply", detail: "was given a null out_error argument", } diff --git a/llama-cpp-bindings/src/send_logs_to_log.rs b/llama-cpp-bindings/src/send_logs_to_log.rs index cf1153d52..82b5f1d09 100644 --- a/llama-cpp-bindings/src/send_logs_to_log.rs +++ b/llama-cpp-bindings/src/send_logs_to_log.rs @@ -14,6 +14,9 @@ use std::ffi::c_void; use std::sync::PoisonError; use std::sync::{Mutex, OnceLock}; +use log::log; +use log::warn; + use llama_cpp_log_decoder::decode_anomaly::DecodeAnomaly; use llama_cpp_log_decoder::decode_output::DecodeOutput; use llama_cpp_log_decoder::incoming_log_level::IncomingLogLevel; @@ -89,7 +92,7 @@ fn resolve_record(line: LogLine, demote_info_to_debug: bool) -> LogRecord { fn dispatch_line(source: &LogSource, line: LogLine) { let LogRecord { level, text } = resolve_record(line, source.options.demote_info_to_debug); - log::log!(target: source.target, level, "{text}"); + log!(target: source.target, level, "{text}"); } fn dispatch_output(source: &LogSource, output: DecodeOutput) { @@ -104,7 +107,7 @@ fn dispatch_output(source: &LogSource, output: DecodeOutput) { } fn dispatch_anomaly(source: &LogSource, anomaly: DecodeAnomaly) { - log::warn!( + warn!( target: source.target, "llama.cpp log decoder anomaly: {anomaly:?}", ); @@ -122,7 +125,7 @@ unsafe extern "C" fn logs_to_log( } if text_ptr.is_null() { - log::warn!( + warn!( target: source.target, "received NULL text pointer from llama.cpp log callback", ); @@ -261,7 +264,7 @@ mod tests { } #[test] - fn test_logger_enabled_and_flush() { + fn the_capturing_logger_enables_every_record() { let metadata = Metadata::builder() .level(Level::Info) .target("test-logger-enabled") diff --git a/llama-cpp-bindings/src/token/data_array.rs b/llama-cpp-bindings/src/token/data_array.rs index 5a5240099..fd1afbb27 100644 --- a/llama-cpp-bindings/src/token/data_array.rs +++ b/llama-cpp-bindings/src/token/data_array.rs @@ -1,5 +1,7 @@ use std::ptr; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use super::LlamaToken; use crate::token::data::LlamaTokenData; @@ -47,7 +49,7 @@ impl LlamaTokenDataArray { impl LlamaTokenDataArray { /// # Errors /// - /// Returns [`llama_cpp_ffi_status::FfiContractError`] when the llama.cpp sampler grows the array beyond the + /// Returns [`llama_cpp_ffi_status::ffi_contract_error::FfiContractError`] when the llama.cpp sampler grows the array beyond the /// capacity this buffer was allocated with, which would make the following `set_len` /// undefined behaviour. /// @@ -59,7 +61,7 @@ impl LlamaTokenDataArray { pub unsafe fn modify_as_c_llama_token_data_array( &mut self, modify: impl FnOnce(&mut llama_cpp_bindings_sys::llama_token_data_array) -> TResult, - ) -> Result { + ) -> Result { let size = self.data.len(); let data = self .data @@ -79,7 +81,7 @@ impl LlamaTokenDataArray { let result = modify(&mut c_llama_token_data_array); if c_llama_token_data_array.size > self.data.capacity() { - return Err(llama_cpp_ffi_status::FfiContractError { + return Err(FfiContractError { operation: "modify_as_c_llama_token_data_array", detail: "the llama.cpp sampler grew the token data array beyond its capacity", }); @@ -109,6 +111,8 @@ impl LlamaTokenDataArray { #[cfg(test)] mod tests { + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use super::LlamaTokenDataArray; use crate::token::LlamaToken; use crate::token::data::LlamaTokenData; @@ -225,7 +229,7 @@ mod tests { assert_eq!( result, - Err(llama_cpp_ffi_status::FfiContractError { + Err(FfiContractError { operation: "modify_as_c_llama_token_data_array", detail: "the llama.cpp sampler grew the token data array beyond its capacity", }) diff --git a/llama-cpp-bindings/src/tool_call_format.rs b/llama-cpp-bindings/src/tool_call_format.rs index 9aabba696..0a67cfbdf 100644 --- a/llama-cpp-bindings/src/tool_call_format.rs +++ b/llama-cpp-bindings/src/tool_call_format.rs @@ -9,8 +9,8 @@ pub mod separator_split; pub mod tool_call_format_outcome; pub mod xml_function_tags; -use llama_cpp_bindings_types::ToolCallArgsShape; -use llama_cpp_bindings_types::ToolCallMarkers; +use llama_cpp_bindings_types::tool_call_args_shape::ToolCallArgsShape; +use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; use crate::error::tool_call_format_failure::ToolCallFormatFailure; use crate::tool_call_format::tool_call_format_outcome::ToolCallFormatOutcome; @@ -48,16 +48,16 @@ pub fn try_parse(body: &str, markers: &ToolCallMarkers) -> ToolCallFormatOutcome mod tests { use serde_json::json; - use llama_cpp_bindings_types::BracketedJsonShape; - use llama_cpp_bindings_types::KeyValueXmlTagsShape; - use llama_cpp_bindings_types::PairedQuoteShape; - use llama_cpp_bindings_types::ParsedToolCall; - use llama_cpp_bindings_types::ParsedToolCallId; - use llama_cpp_bindings_types::ToolCallArgsShape; - use llama_cpp_bindings_types::ToolCallArguments; - use llama_cpp_bindings_types::ToolCallMarkers; - use llama_cpp_bindings_types::ToolCallValueQuote; - use llama_cpp_bindings_types::XmlTagsShape; + use llama_cpp_bindings_types::bracketed_json_shape::BracketedJsonShape; + use llama_cpp_bindings_types::key_value_xml_tags_shape::KeyValueXmlTagsShape; + use llama_cpp_bindings_types::paired_quote_shape::PairedQuoteShape; + use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; + use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; + use llama_cpp_bindings_types::tool_call_args_shape::ToolCallArgsShape; + use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; + use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; + use llama_cpp_bindings_types::tool_call_value_quote::ToolCallValueQuote; + use llama_cpp_bindings_types::xml_tags_shape::XmlTagsShape; use super::ToolCallFormatOutcome; use super::try_parse; diff --git a/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs b/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs index 4b85a4921..e982c55d5 100644 --- a/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs +++ b/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs @@ -1,8 +1,8 @@ -use llama_cpp_bindings_types::BracketedJsonShape; -use llama_cpp_bindings_types::ParsedToolCall; -use llama_cpp_bindings_types::ParsedToolCallId; -use llama_cpp_bindings_types::ToolCallArguments; -use llama_cpp_bindings_types::ToolCallMarkers; +use llama_cpp_bindings_types::bracketed_json_shape::BracketedJsonShape; +use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; +use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; +use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; +use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; use crate::error::bracketed_args_failure::BracketedArgsFailure; use crate::tool_call_format::consume_optional_prefix::consume_optional_prefix; @@ -102,11 +102,11 @@ mod tests { use serde_json::json; - use llama_cpp_bindings_types::BracketedJsonShape; - use llama_cpp_bindings_types::ParsedToolCall; - use llama_cpp_bindings_types::ToolCallArgsShape; - use llama_cpp_bindings_types::ToolCallArguments; - use llama_cpp_bindings_types::ToolCallMarkers; + use llama_cpp_bindings_types::bracketed_json_shape::BracketedJsonShape; + use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; + use llama_cpp_bindings_types::tool_call_args_shape::ToolCallArgsShape; + use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; + use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; use super::parse; use crate::error::bracketed_args_failure::BracketedArgsFailure; diff --git a/llama-cpp-bindings/src/tool_call_format/json_object.rs b/llama-cpp-bindings/src/tool_call_format/json_object.rs index 5c9ab8d4a..3b2d75a2e 100644 --- a/llama-cpp-bindings/src/tool_call_format/json_object.rs +++ b/llama-cpp-bindings/src/tool_call_format/json_object.rs @@ -1,7 +1,7 @@ -use llama_cpp_bindings_types::JsonObjectShape; -use llama_cpp_bindings_types::ParsedToolCall; -use llama_cpp_bindings_types::ParsedToolCallId; -use llama_cpp_bindings_types::ToolCallArguments; +use llama_cpp_bindings_types::json_object_shape::JsonObjectShape; +use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; +use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; +use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; use crate::error::json_object_failure::JsonObjectFailure; @@ -81,9 +81,9 @@ pub fn parse( mod tests { use serde_json::json; - use llama_cpp_bindings_types::JsonObjectShape; - use llama_cpp_bindings_types::ParsedToolCall; - use llama_cpp_bindings_types::ToolCallArguments; + use llama_cpp_bindings_types::json_object_shape::JsonObjectShape; + use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; + use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; use super::parse; use crate::error::json_object_failure::JsonObjectFailure; diff --git a/llama-cpp-bindings/src/tool_call_format/key_value_xml_tags.rs b/llama-cpp-bindings/src/tool_call_format/key_value_xml_tags.rs index c5d635f39..dedae7f91 100644 --- a/llama-cpp-bindings/src/tool_call_format/key_value_xml_tags.rs +++ b/llama-cpp-bindings/src/tool_call_format/key_value_xml_tags.rs @@ -2,11 +2,11 @@ use nom::IResult; use nom::Parser; use nom::bytes::complete::take_until; -use llama_cpp_bindings_types::KeyValueXmlTagsShape; -use llama_cpp_bindings_types::ParsedToolCall; -use llama_cpp_bindings_types::ParsedToolCallId; -use llama_cpp_bindings_types::ToolCallArguments; -use llama_cpp_bindings_types::ToolCallMarkers; +use llama_cpp_bindings_types::key_value_xml_tags_shape::KeyValueXmlTagsShape; +use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; +use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; +use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; +use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; use crate::error::key_value_xml_tags_failure::KeyValueXmlTagsFailure; use crate::tool_call_format::parse_step::ParseStep; @@ -171,11 +171,11 @@ pub fn parse( mod tests { use serde_json::json; - use llama_cpp_bindings_types::KeyValueXmlTagsShape; - use llama_cpp_bindings_types::ParsedToolCall; - use llama_cpp_bindings_types::ToolCallArgsShape; - use llama_cpp_bindings_types::ToolCallArguments; - use llama_cpp_bindings_types::ToolCallMarkers; + use llama_cpp_bindings_types::key_value_xml_tags_shape::KeyValueXmlTagsShape; + use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; + use llama_cpp_bindings_types::tool_call_args_shape::ToolCallArgsShape; + use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; + use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; use super::parse; use crate::error::key_value_xml_tags_failure::KeyValueXmlTagsFailure; diff --git a/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs b/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs index b8fdbf856..ce7811d50 100644 --- a/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs +++ b/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs @@ -1,9 +1,9 @@ -use llama_cpp_bindings_types::PairedQuoteShape; -use llama_cpp_bindings_types::ParsedToolCall; -use llama_cpp_bindings_types::ParsedToolCallId; -use llama_cpp_bindings_types::ToolCallArguments; -use llama_cpp_bindings_types::ToolCallMarkers; -use llama_cpp_bindings_types::ToolCallValueQuote; +use llama_cpp_bindings_types::paired_quote_shape::PairedQuoteShape; +use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; +use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; +use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; +use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; +use llama_cpp_bindings_types::tool_call_value_quote::ToolCallValueQuote; use crate::error::paired_quote_failure::PairedQuoteFailure; use crate::tool_call_format::consume_optional_prefix::consume_optional_prefix; @@ -208,13 +208,13 @@ pub fn parse( mod tests { use serde_json::json; - use llama_cpp_bindings_types::PairedQuoteShape; - use llama_cpp_bindings_types::ParsedToolCall; - use llama_cpp_bindings_types::ParsedToolCallId; - use llama_cpp_bindings_types::ToolCallArgsShape; - use llama_cpp_bindings_types::ToolCallArguments; - use llama_cpp_bindings_types::ToolCallMarkers; - use llama_cpp_bindings_types::ToolCallValueQuote; + use llama_cpp_bindings_types::paired_quote_shape::PairedQuoteShape; + use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; + use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; + use llama_cpp_bindings_types::tool_call_args_shape::ToolCallArgsShape; + use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; + use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; + use llama_cpp_bindings_types::tool_call_value_quote::ToolCallValueQuote; use super::parse; use crate::error::paired_quote_failure::PairedQuoteFailure; diff --git a/llama-cpp-bindings/src/tool_call_format/parse_step.rs b/llama-cpp-bindings/src/tool_call_format/parse_step.rs index 70eb59711..f2005932f 100644 --- a/llama-cpp-bindings/src/tool_call_format/parse_step.rs +++ b/llama-cpp-bindings/src/tool_call_format/parse_step.rs @@ -1,4 +1,4 @@ -use llama_cpp_bindings_types::ParsedToolCall; +use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; #[derive(Debug)] pub enum ParseStep<'body> { diff --git a/llama-cpp-bindings/src/tool_call_format/tool_call_format_outcome.rs b/llama-cpp-bindings/src/tool_call_format/tool_call_format_outcome.rs index e026529d6..17f9ab868 100644 --- a/llama-cpp-bindings/src/tool_call_format/tool_call_format_outcome.rs +++ b/llama-cpp-bindings/src/tool_call_format/tool_call_format_outcome.rs @@ -1,4 +1,4 @@ -use llama_cpp_bindings_types::ParsedToolCall; +use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; use crate::error::tool_call_format_failure::ToolCallFormatFailure; diff --git a/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs b/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs index ce5d10295..4ffd31624 100644 --- a/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs +++ b/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs @@ -2,10 +2,10 @@ use nom::IResult; use nom::Parser; use nom::bytes::complete::take_until; -use llama_cpp_bindings_types::ParsedToolCall; -use llama_cpp_bindings_types::ParsedToolCallId; -use llama_cpp_bindings_types::ToolCallArguments; -use llama_cpp_bindings_types::XmlTagsShape; +use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; +use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; +use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; +use llama_cpp_bindings_types::xml_tags_shape::XmlTagsShape; use crate::error::xml_function_tags_failure::XmlFunctionTagsFailure; use crate::tool_call_format::scalar_value_to_json::scalar_value_to_json; @@ -177,9 +177,9 @@ pub fn parse( mod tests { use serde_json::json; - use llama_cpp_bindings_types::ParsedToolCall; - use llama_cpp_bindings_types::ToolCallArguments; - use llama_cpp_bindings_types::XmlTagsShape; + use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; + use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; + use llama_cpp_bindings_types::xml_tags_shape::XmlTagsShape; use super::parse; use crate::error::xml_function_tags_failure::XmlFunctionTagsFailure; diff --git a/llama-cpp-error-recorder/src/frame_stack.rs b/llama-cpp-error-recorder/src/frame_stack.rs index e011c66da..f20262b12 100644 --- a/llama-cpp-error-recorder/src/frame_stack.rs +++ b/llama-cpp-error-recorder/src/frame_stack.rs @@ -1,5 +1,7 @@ use std::cell::RefCell; +use log::error; + use crate::recorded_error::RecordedError; thread_local! { @@ -27,15 +29,13 @@ impl FrameStack { FRAMES.with(|cell| { let mut frames = cell.borrow_mut(); let Some(top) = frames.last_mut() else { - log::error!("an FFI callback failed outside any error scope: {error}"); + error!("an FFI callback failed outside any error scope: {error}"); return; }; if top.is_some() { - log::error!( - "an FFI callback failed again after the root cause was recorded: {error}" - ); + error!("an FFI callback failed again after the root cause was recorded: {error}"); return; } diff --git a/llama-cpp-error-recorder/src/lib.rs b/llama-cpp-error-recorder/src/lib.rs index 42dcf41a8..b7a8e7e7e 100644 --- a/llama-cpp-error-recorder/src/lib.rs +++ b/llama-cpp-error-recorder/src/lib.rs @@ -8,7 +8,3 @@ mod frame_stack; pub mod error_scope; pub mod record; pub mod recorded_error; - -pub use error_scope::ErrorScope; -pub use record::record; -pub use recorded_error::RecordedError; diff --git a/llama-cpp-fault-injection/src/lib.rs b/llama-cpp-fault-injection/src/lib.rs index 9dc8ab6bb..18526e8b3 100644 --- a/llama-cpp-fault-injection/src/lib.rs +++ b/llama-cpp-fault-injection/src/lib.rs @@ -1,5 +1,2 @@ pub mod armed_fault; pub mod injected_fault_message; - -pub use armed_fault::ArmedFault; -pub use injected_fault_message::injected_fault_message; diff --git a/llama-cpp-ffi-status/src/lib.rs b/llama-cpp-ffi-status/src/lib.rs index c1ab08546..84bbf604a 100644 --- a/llama-cpp-ffi-status/src/lib.rs +++ b/llama-cpp-ffi-status/src/lib.rs @@ -11,7 +11,3 @@ pub mod ffi_contract_error; pub mod ffi_status_error; pub mod read_and_free_cpp_string; - -pub use ffi_contract_error::FfiContractError; -pub use ffi_status_error::FfiStatusError; -pub use read_and_free_cpp_string::read_and_free_cpp_string; diff --git a/llama-cpp-ffi-status/src/read_and_free_cpp_string.rs b/llama-cpp-ffi-status/src/read_and_free_cpp_string.rs index 363cf2df1..9a92bf2ba 100644 --- a/llama-cpp-ffi-status/src/read_and_free_cpp_string.rs +++ b/llama-cpp-ffi-status/src/read_and_free_cpp_string.rs @@ -41,7 +41,7 @@ pub unsafe fn read_and_free_cpp_string( mod tests { use std::ptr; - use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer; + use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer::wrapper_allocated_error_message_pointer; use super::read_and_free_cpp_string; use crate::ffi_contract_error::FfiContractError; diff --git a/llama-cpp-gbnf/src/gbnf_validation_error.rs b/llama-cpp-gbnf/src/gbnf_validation_error.rs index 2d0097b52..2b4414904 100644 --- a/llama-cpp-gbnf/src/gbnf_validation_error.rs +++ b/llama-cpp-gbnf/src/gbnf_validation_error.rs @@ -1,11 +1,14 @@ use std::ffi::NulError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, thiserror::Error, PartialEq, Eq)] pub enum GbnfValidationError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("grammar string contains an interior NUL byte")] GrammarContainsNul(#[source] NulError), #[error("grammar root name contains an interior NUL byte")] diff --git a/llama-cpp-gbnf/src/validate_gbnf.rs b/llama-cpp-gbnf/src/validate_gbnf.rs index 2c8fcd992..db255f476 100644 --- a/llama-cpp-gbnf/src/validate_gbnf.rs +++ b/llama-cpp-gbnf/src/validate_gbnf.rs @@ -10,9 +10,9 @@ use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_UNSUPPORTED_GRAMMAR; use llama_cpp_bindings_sys::llama_rs_gbnf_validation_status; use llama_cpp_bindings_sys::llama_rs_validate_gbnf; use llama_cpp_bindings_sys::llama_vocab; -use llama_cpp_ffi_status::FfiContractError; -use llama_cpp_ffi_status::FfiStatusError; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use crate::gbnf_validation_error::GbnfValidationError; @@ -109,8 +109,8 @@ mod tests { use std::ptr; use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_THREW_CXX_EXCEPTION; - use llama_cpp_ffi_status::FfiContractError; - use llama_cpp_ffi_status::FfiStatusError; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; use super::validate_gbnf; use super::validation_status_to_result; diff --git a/llama-cpp-log-decoder/src/log_decoder.rs b/llama-cpp-log-decoder/src/log_decoder.rs index fba0ba0a7..166db2559 100644 --- a/llama-cpp-log-decoder/src/log_decoder.rs +++ b/llama-cpp-log-decoder/src/log_decoder.rs @@ -414,7 +414,7 @@ mod tests { } #[test] - fn default_construction() { + fn a_default_decoder_decodes_like_a_new_one() { let mut default_decoder = LogDecoder::default(); let new_decoder_result = LogDecoder::new().feed(IncomingLogLevel::Info, "compare\n"); let default_result = default_decoder.feed(IncomingLogLevel::Info, "compare\n"); diff --git a/llama-cpp-test-harness-macros/src/expand.rs b/llama-cpp-test-harness-macros/src/expand.rs index 52e99c909..cdcbc9e42 100644 --- a/llama-cpp-test-harness-macros/src/expand.rs +++ b/llama-cpp-test-harness-macros/src/expand.rs @@ -73,13 +73,13 @@ fn split_fn_and_pass_through(items: Vec) -> syn::Result<(ItemFn, Vec fn build_model_source_literal(source: &ParsedSource) -> TokenStream { match source { ParsedSource::HuggingFace { repo, file } => quote! { - ::llama_cpp_test_harness::GgufSource::HuggingFace { + ::llama_cpp_test_harness::gguf_source::GgufSource::HuggingFace { repo: #repo, file: #file, } }, ParsedSource::LocalPath(path) => quote! { - ::llama_cpp_test_harness::GgufSource::LocalPath(#path) + ::llama_cpp_test_harness::gguf_source::GgufSource::LocalPath(#path) }, } } @@ -88,13 +88,13 @@ fn build_mmproj_source_literal(source: Option<&ParsedSource>) -> TokenStream { match source { None => quote! { ::core::option::Option::None }, Some(ParsedSource::HuggingFace { repo, file }) => quote! { - ::core::option::Option::Some(::llama_cpp_test_harness::GgufSource::HuggingFace { + ::core::option::Option::Some(::llama_cpp_test_harness::gguf_source::GgufSource::HuggingFace { repo: #repo, file: #file, }) }, Some(ParsedSource::LocalPath(path)) => quote! { - ::core::option::Option::Some(::llama_cpp_test_harness::GgufSource::LocalPath(#path)) + ::core::option::Option::Some(::llama_cpp_test_harness::gguf_source::GgufSource::LocalPath(#path)) }, } } @@ -104,10 +104,10 @@ fn build_test_fn_literal(signature: &Signature) -> TokenStream { match signature.output { ReturnType::Default => { - quote! { ::llama_cpp_test_harness::LlamaTestFn::Infallible(#fn_name) } + quote! { ::llama_cpp_test_harness::llama_test_fn::LlamaTestFn::Infallible(#fn_name) } } ReturnType::Type(..) => { - quote! { ::llama_cpp_test_harness::LlamaTestFn::Fallible(#fn_name) } + quote! { ::llama_cpp_test_harness::llama_test_fn::LlamaTestFn::Fallible(#fn_name) } } } } @@ -135,18 +135,18 @@ fn build_registration(args: &ParsedArgs, signature: &Signature) -> TokenStream { ); quote! { - ::llama_cpp_test_harness::inventory::submit! { - ::llama_cpp_test_harness::LlamaTestRegistration { + ::inventory::submit! { + ::llama_cpp_test_harness::llama_test_registration::LlamaTestRegistration { name: #trial_name, - key: ::llama_cpp_test_harness::LoadKey { + key: ::llama_cpp_test_harness::load_key::LoadKey { model_source: #model_source_literal, mmproj_source: #mmproj_source_literal, - model_load_params: ::llama_cpp_test_harness::ModelLoadParams { + model_load_params: ::llama_cpp_test_harness::model_load_params::ModelLoadParams { n_gpu_layers: #gpu_layers, load_mode: #load_mode, }, }, - context_params: ::llama_cpp_test_harness::ContextParams { + context_params: ::llama_cpp_test_harness::context_params::ContextParams { n_ctx: #context_size, n_batch: #logical_batch, n_ubatch: #physical_batch, @@ -226,21 +226,21 @@ mod tests { .expect("well-formed input must expand"); let expected = quote! { fn my_test(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } - ::llama_cpp_test_harness::inventory::submit! { - ::llama_cpp_test_harness::LlamaTestRegistration { + ::inventory::submit! { + ::llama_cpp_test_harness::llama_test_registration::LlamaTestRegistration { name: "my_test[bar.gguf]", - key: ::llama_cpp_test_harness::LoadKey { - model_source: ::llama_cpp_test_harness::GgufSource::HuggingFace { + key: ::llama_cpp_test_harness::load_key::LoadKey { + model_source: ::llama_cpp_test_harness::gguf_source::GgufSource::HuggingFace { repo: "foo", file: "bar.gguf", }, mmproj_source: ::core::option::Option::None, - model_load_params: ::llama_cpp_test_harness::ModelLoadParams { + model_load_params: ::llama_cpp_test_harness::model_load_params::ModelLoadParams { n_gpu_layers: 0i32, - load_mode: ::llama_cpp_test_harness::LlamaLoadMode::Mmap, + load_mode: ::llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode::Mmap, }, }, - context_params: ::llama_cpp_test_harness::ContextParams { + context_params: ::llama_cpp_test_harness::context_params::ContextParams { n_ctx: 1u32, n_batch: 1u32, n_ubatch: 1u32, @@ -249,7 +249,7 @@ mod tests { embeddings: false, }, void_logs: false, - func: ::llama_cpp_test_harness::LlamaTestFn::Fallible(my_test), + func: ::llama_cpp_test_harness::llama_test_fn::LlamaTestFn::Fallible(my_test), } } }; @@ -261,7 +261,8 @@ mod tests { fn a_function_returning_a_result_registers_as_fallible() { assert_eq!( test_fn_literal_of(well_formed_function()), - quote! { ::llama_cpp_test_harness::LlamaTestFn::Fallible(my_test) }.to_string() + quote! { ::llama_cpp_test_harness::llama_test_fn::LlamaTestFn::Fallible(my_test) } + .to_string() ); } @@ -269,7 +270,8 @@ mod tests { fn a_function_without_a_return_type_registers_as_infallible() { assert_eq!( test_fn_literal_of(quote! { fn my_test(fixture: &LlamaFixture<'_>) {} }), - quote! { ::llama_cpp_test_harness::LlamaTestFn::Infallible(my_test) }.to_string() + quote! { ::llama_cpp_test_harness::llama_test_fn::LlamaTestFn::Infallible(my_test) } + .to_string() ); } @@ -278,7 +280,7 @@ mod tests { assert_eq!( build_model_source_literal(&ParsedSource::LocalPath("/abs/local.gguf".to_owned())) .to_string(), - quote! { ::llama_cpp_test_harness::GgufSource::LocalPath("/abs/local.gguf") } + quote! { ::llama_cpp_test_harness::gguf_source::GgufSource::LocalPath("/abs/local.gguf") } .to_string() ); } @@ -292,7 +294,7 @@ mod tests { })) .to_string(), quote! { - ::core::option::Option::Some(::llama_cpp_test_harness::GgufSource::HuggingFace { + ::core::option::Option::Some(::llama_cpp_test_harness::gguf_source::GgufSource::HuggingFace { repo: "r", file: "mmproj.gguf", }) @@ -309,7 +311,7 @@ mod tests { ))) .to_string(), quote! { - ::core::option::Option::Some(::llama_cpp_test_harness::GgufSource::LocalPath( + ::core::option::Option::Some(::llama_cpp_test_harness::gguf_source::GgufSource::LocalPath( "/abs/mmproj.gguf" )) } diff --git a/llama-cpp-test-harness-macros/src/parsed_load_mode.rs b/llama-cpp-test-harness-macros/src/parsed_load_mode.rs index 9cac56011..cf9a85460 100644 --- a/llama-cpp-test-harness-macros/src/parsed_load_mode.rs +++ b/llama-cpp-test-harness-macros/src/parsed_load_mode.rs @@ -30,12 +30,24 @@ impl ParsedLoadMode { pub fn tokens(self) -> TokenStream { match self { - Self::Auto => quote! { ::llama_cpp_test_harness::LlamaLoadMode::Auto }, - Self::None => quote! { ::llama_cpp_test_harness::LlamaLoadMode::None }, - Self::Mmap => quote! { ::llama_cpp_test_harness::LlamaLoadMode::Mmap }, - Self::Mlock => quote! { ::llama_cpp_test_harness::LlamaLoadMode::Mlock }, - Self::MmapMlock => quote! { ::llama_cpp_test_harness::LlamaLoadMode::MmapMlock }, - Self::DirectIo => quote! { ::llama_cpp_test_harness::LlamaLoadMode::DirectIo }, + Self::Auto => { + quote! { ::llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode::Auto } + } + Self::None => { + quote! { ::llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode::None } + } + Self::Mmap => { + quote! { ::llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode::Mmap } + } + Self::Mlock => { + quote! { ::llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode::Mlock } + } + Self::MmapMlock => { + quote! { ::llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode::MmapMlock } + } + Self::DirectIo => { + quote! { ::llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode::DirectIo } + } } } } @@ -65,7 +77,9 @@ mod tests { assert_eq!(parsed, expected); assert_eq!( parsed.tokens().to_string(), - format!(":: llama_cpp_test_harness :: LlamaLoadMode :: {identifier}"), + format!( + ":: llama_cpp_bindings :: model :: llama_load_mode :: LlamaLoadMode :: {identifier}" + ), ); } } diff --git a/llama-cpp-test-harness/Cargo.toml b/llama-cpp-test-harness/Cargo.toml index 5089cdc02..d40cb3792 100644 --- a/llama-cpp-test-harness/Cargo.toml +++ b/llama-cpp-test-harness/Cargo.toml @@ -23,7 +23,7 @@ vulkan = ["llama-cpp-bindings/vulkan"] rocm = ["llama-cpp-bindings/rocm"] [[test]] -name = "harness_self_test" +name = "phase_conclusions" harness = false [lints] diff --git a/llama-cpp-test-harness/src/execution_phase.rs b/llama-cpp-test-harness/src/execution_phase.rs index be607f872..02fab4930 100644 --- a/llama-cpp-test-harness/src/execution_phase.rs +++ b/llama-cpp-test-harness/src/execution_phase.rs @@ -26,9 +26,10 @@ impl ExecutionPhase { #[cfg(test)] mod tests { + use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; + use super::ExecutionPhase; - use crate::GgufSource; - use crate::LlamaLoadMode; + use crate::gguf_source::GgufSource; use crate::load_key::LoadKey; use crate::model_load_params::ModelLoadParams; diff --git a/llama-cpp-test-harness/src/execution_plan.rs b/llama-cpp-test-harness/src/execution_plan.rs index e52a23c7d..4ae45f60e 100644 --- a/llama-cpp-test-harness/src/execution_plan.rs +++ b/llama-cpp-test-harness/src/execution_plan.rs @@ -74,8 +74,9 @@ impl ExecutionPlan { #[cfg(test)] mod tests { + use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; + use super::ExecutionPlan; - use crate::LlamaLoadMode; use crate::context_params::ContextParams; use crate::gguf_source::GgufSource; use crate::llama_test_fn::LlamaTestFn; diff --git a/llama-cpp-test-harness/src/lib.rs b/llama-cpp-test-harness/src/lib.rs index bbef25d9e..c41df52fb 100644 --- a/llama-cpp-test-harness/src/lib.rs +++ b/llama-cpp-test-harness/src/lib.rs @@ -26,26 +26,3 @@ pub mod run_to_conclusions; pub mod scheduled_phase; #[cfg(test)] mod test_backend_gate; - -#[doc(hidden)] -pub use inventory; - -pub use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; -pub use llama_cpp_test_harness_macros::llama_test; - -pub use crate::context_params::ContextParams; -pub use crate::execution_phase::ExecutionPhase; -pub use crate::execution_plan::ExecutionPlan; -pub use crate::gguf_source::GgufSource; -pub use crate::llama_fixture::LlamaFixture; -pub use crate::llama_test_fn::LlamaTestFn; -pub use crate::llama_test_registration::LlamaTestRegistration; -pub use crate::load_key::LoadKey; -pub use crate::model_load_params::ModelLoadParams; -pub use crate::no_op::no_op; -pub use crate::phase_model_files::PhaseModelFiles; -pub use crate::phase_setup::PhaseSetup; -pub use crate::phase_state::PhaseState; -pub use crate::run::run; -pub use crate::run_to_conclusions::run_to_conclusions; -pub use crate::scheduled_phase::ScheduledPhase; diff --git a/llama-cpp-test-harness/src/llama_test_registration.rs b/llama-cpp-test-harness/src/llama_test_registration.rs index 2105120d5..80f24f5da 100644 --- a/llama-cpp-test-harness/src/llama_test_registration.rs +++ b/llama-cpp-test-harness/src/llama_test_registration.rs @@ -1,3 +1,5 @@ +use inventory::collect; + use crate::context_params::ContextParams; use crate::llama_test_fn::LlamaTestFn; use crate::load_key::LoadKey; @@ -10,4 +12,4 @@ pub struct LlamaTestRegistration { pub func: LlamaTestFn, } -inventory::collect!(LlamaTestRegistration); +collect!(LlamaTestRegistration); diff --git a/llama-cpp-test-harness/src/llama_tests_main_macro.rs b/llama-cpp-test-harness/src/llama_tests_main_macro.rs index 18ed6dab0..5ae5434a5 100644 --- a/llama-cpp-test-harness/src/llama_tests_main_macro.rs +++ b/llama-cpp-test-harness/src/llama_tests_main_macro.rs @@ -2,7 +2,7 @@ macro_rules! llama_tests_main { () => { fn main() -> ::std::process::ExitCode { - $crate::run() + $crate::run::run() } }; } diff --git a/llama-cpp-test-harness/src/run.rs b/llama-cpp-test-harness/src/run.rs index 5a9564648..f1e9e0ed3 100644 --- a/llama-cpp-test-harness/src/run.rs +++ b/llama-cpp-test-harness/src/run.rs @@ -86,7 +86,7 @@ mod tests { } #[test] - fn aggregate_exit_code_empty_input_succeeds() { + fn aggregate_exit_code_reports_success_when_no_phase_ran() { let code = aggregate_exit_code(&[]); assert_eq!(as_u8(code), 0); diff --git a/llama-cpp-test-harness/tests/harness_self_test.rs b/llama-cpp-test-harness/tests/phase_conclusions.rs similarity index 92% rename from llama-cpp-test-harness/tests/harness_self_test.rs rename to llama-cpp-test-harness/tests/phase_conclusions.rs index d0cec11b6..28c61c24f 100644 --- a/llama-cpp-test-harness/tests/harness_self_test.rs +++ b/llama-cpp-test-harness/tests/phase_conclusions.rs @@ -4,10 +4,10 @@ use anyhow::Result; use anyhow::bail; use libtest_mimic::Arguments; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; -use llama_cpp_test_harness::no_op; -use llama_cpp_test_harness::run_to_conclusions; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness::no_op::no_op; +use llama_cpp_test_harness::run_to_conclusions::run_to_conclusions; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), @@ -142,7 +142,7 @@ fn main() -> ExitCode { let conclusions = match run_to_conclusions(Arguments::from_args()) { Ok(conclusions) => conclusions, Err(error) => { - eprintln!("harness_self_test: unexpected harness setup failure: {error}"); + eprintln!("phase_conclusions: unexpected harness setup failure: {error}"); return ExitCode::FAILURE; } @@ -162,12 +162,12 @@ fn main() -> ExitCode { && total_failed == EXPECTED_FAILED { eprintln!( - "harness_self_test: as expected — phases={phases}, passed={total_passed}, failed={total_failed}" + "phase_conclusions: as expected — phases={phases}, passed={total_passed}, failed={total_failed}" ); ExitCode::SUCCESS } else { eprintln!( - "harness_self_test: UNEXPECTED — phases={phases} (want {EXPECTED_PHASES}), \ + "phase_conclusions: UNEXPECTED — phases={phases} (want {EXPECTED_PHASES}), \ passed={total_passed} (want {EXPECTED_PASSED}), \ failed={total_failed} (want {EXPECTED_FAILED})" ); diff --git a/llama-cpp-wrapper-error-fixture/src/lib.rs b/llama-cpp-wrapper-error-fixture/src/lib.rs index fb54cf139..6e5f36bec 100644 --- a/llama-cpp-wrapper-error-fixture/src/lib.rs +++ b/llama-cpp-wrapper-error-fixture/src/lib.rs @@ -1,3 +1 @@ pub mod wrapper_allocated_error_message_pointer; - -pub use wrapper_allocated_error_message_pointer::wrapper_allocated_error_message_pointer; From cf6016b1c3e3cd916f24abb8a86d08ec58a01236 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sat, 31 Oct 2026 19:23:40 +0100 Subject: [PATCH 30/82] Look up chat span roles by binary search over segments, release each piece with its one allocation, and name every test after what it verifies --- .../tests/context_dimensions.rs | 11 -- .../tests/context_synchronization.rs | 2 +- .../tests/detokenization.rs | 4 +- .../tests/embedding_generation.rs | 8 +- .../tests/encoder_requirement.rs | 2 +- .../json_schema_constrained_generation.rs | 4 +- .../tests/logit_bias_sampler.rs | 2 +- .../tests/logits_retrieval.rs | 2 +- .../tests/model_hyperparameters.rs | 151 +--------------- .../multimodal_projector_capabilities.rs | 10 +- .../tests/parameter_fitting.rs | 2 +- .../tests/sampler_token_acceptance.rs | 21 +-- .../tests/session_files.rs | 8 +- .../tests/state_snapshots.rs | 4 +- .../tests/threadpool_detachment.rs | 2 +- .../tests/token_candidates.rs | 4 +- .../tests/tokenization.rs | 4 +- .../tests/unconstrained_token_sampling.rs | 61 +------ .../tests/vocabulary_enumeration.rs | 2 +- llama-cpp-bindings/src/chat_output.rs | 51 ++---- llama-cpp-bindings/src/chat_session.rs | 6 +- llama-cpp-bindings/src/chat_span_segment.rs | 9 + llama-cpp-bindings/src/chat_spans.rs | 161 ++++++++++++------ .../src/context/llama_attention_type.rs | 2 +- .../src/context/llama_pooling_type.rs | 2 +- llama-cpp-bindings/src/context/params.rs | 2 +- .../src/context/rope_scaling_type.rs | 2 +- llama-cpp-bindings/src/gguf_type.rs | 2 +- llama-cpp-bindings/src/lib.rs | 3 +- llama-cpp-bindings/src/llama_backend.rs | 32 +++- .../src/llama_backend_device.rs | 2 +- .../src/llama_backend_numa_strategy.rs | 2 +- llama-cpp-bindings/src/llama_time_us.rs | 2 +- llama-cpp-bindings/src/max_devices.rs | 2 +- .../src/model/llama_chat_message.rs | 2 +- .../src/model/llama_chat_template.rs | 6 +- llama-cpp-bindings/src/model/params.rs | 9 +- .../src/model/params/kv_overrides.rs | 9 - .../src/model/params/param_override_value.rs | 8 +- llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs | 6 +- .../src/mtmd/mtmd_input_chunk.rs | 6 +- ..._output_token.rs => pending_chat_piece.rs} | 5 +- .../src/sampled_token_classifier.rs | 33 ++-- llama-cpp-bindings/src/sampling.rs | 29 ++-- llama-cpp-bindings/src/send_logs_to_log.rs | 20 +-- .../src/incoming_log_level.rs | 2 +- llama-cpp-log-decoder/src/log_decoder.rs | 101 ++++------- llama-cpp-log-decoder/src/log_level.rs | 2 +- 48 files changed, 304 insertions(+), 518 deletions(-) create mode 100644 llama-cpp-bindings/src/chat_span_segment.rs rename llama-cpp-bindings/src/{chat_output_token.rs => pending_chat_piece.rs} (61%) diff --git a/llama-cpp-bindings-tests/tests/context_dimensions.rs b/llama-cpp-bindings-tests/tests/context_dimensions.rs index f2dedfe84..0174e208b 100644 --- a/llama-cpp-bindings-tests/tests/context_dimensions.rs +++ b/llama-cpp-bindings-tests/tests/context_dimensions.rs @@ -36,17 +36,6 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 128, n_ubatch = 64, )] -fn new_context_returns_valid_context(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - assert!(context.n_ctx() > 0); - Ok(()) -} - #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, diff --git a/llama-cpp-bindings-tests/tests/context_synchronization.rs b/llama-cpp-bindings-tests/tests/context_synchronization.rs index 11c216b72..435d3c84d 100644 --- a/llama-cpp-bindings-tests/tests/context_synchronization.rs +++ b/llama-cpp-bindings-tests/tests/context_synchronization.rs @@ -12,7 +12,7 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 2048, n_ubatch = 512, )] -fn synchronize_completes_without_panic(fixture: &LlamaFixture<'_>) -> Result<()> { +fn an_idle_context_synchronizes(fixture: &LlamaFixture<'_>) -> Result<()> { let context = LlamaContext::from_model( fixture.model, fixture.backend, diff --git a/llama-cpp-bindings-tests/tests/detokenization.rs b/llama-cpp-bindings-tests/tests/detokenization.rs index 497c7cd36..c3c0f47b4 100644 --- a/llama-cpp-bindings-tests/tests/detokenization.rs +++ b/llama-cpp-bindings-tests/tests/detokenization.rs @@ -174,7 +174,9 @@ fn token_to_piece_bytes_insufficient_buffer_returns_error( n_batch = 128, n_ubatch = 64, )] -fn token_to_piece_with_lstrip_succeeds(fixture: &LlamaFixture<'_>) -> Result<()> { +fn token_to_piece_accepts_a_count_of_leading_spaces_to_strip( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let model = fixture.model; let mut decoder = encoding_rs::UTF_8.new_decoder(); let tokens = model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; diff --git a/llama-cpp-bindings-tests/tests/embedding_generation.rs b/llama-cpp-bindings-tests/tests/embedding_generation.rs index f14843fde..619640ac4 100644 --- a/llama-cpp-bindings-tests/tests/embedding_generation.rs +++ b/llama-cpp-bindings-tests/tests/embedding_generation.rs @@ -91,7 +91,7 @@ fn embedding_generation_produces_vectors(fixture: &LlamaFixture<'_>) -> Result<( n_ubatch = 512, embeddings = true, )] -fn decoding_with_embeddings_enabled_succeeds(fixture: &LlamaFixture<'_>) -> Result<()> { +fn an_embeddings_context_decodes_a_prompt(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = LlamaContext::from_model( fixture.model, fixture.backend, @@ -119,7 +119,9 @@ fn decoding_with_embeddings_enabled_succeeds(fixture: &LlamaFixture<'_>) -> Resu n_ubatch = 512, embeddings = true, )] -fn embeddings_seq_ith_returns_valid_embeddings(fixture: &LlamaFixture<'_>) -> Result<()> { +fn a_sequence_embedding_is_as_wide_as_the_model_embedding( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let mut context = LlamaContext::from_model( fixture.model, fixture.backend, @@ -297,7 +299,7 @@ fn embeddings_returns_distinct_values_when_reused_batch_has_extra_capacity( n_ubatch = 512, embeddings = true, )] -fn embeddings_ith_returns_valid_embeddings(fixture: &LlamaFixture<'_>) -> Result<()> { +fn a_token_embedding_is_as_wide_as_the_model_embedding(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = LlamaContext::from_model( fixture.model, fixture.backend, diff --git a/llama-cpp-bindings-tests/tests/encoder_requirement.rs b/llama-cpp-bindings-tests/tests/encoder_requirement.rs index 5f5f99129..8139f97f2 100644 --- a/llama-cpp-bindings-tests/tests/encoder_requirement.rs +++ b/llama-cpp-bindings-tests/tests/encoder_requirement.rs @@ -45,7 +45,7 @@ fn encode_on_non_encoder_model_returns_error(fixture: &LlamaFixture<'_>) -> Resu n_ubatch = 512, embeddings = true, )] -fn encode_succeeds_with_encoder_model(fixture: &LlamaFixture<'_>) -> Result<()> { +fn an_encoder_model_encodes_a_prompt(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = LlamaContext::from_model( fixture.model, fixture.backend, diff --git a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs index fceee06ab..0d6400fd9 100644 --- a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs +++ b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs @@ -136,7 +136,9 @@ fn json_schema_grammar_sampler_constrains_output_to_json(fixture: &LlamaFixture< n_batch = 512, n_ubatch = 128, )] -fn json_schema_constrains_output(fixture: &LlamaFixture<'_>) -> Result<()> { +fn an_llguidance_json_schema_constrains_the_answer_to_the_schema( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let model = fixture.model; let backend = fixture.backend; diff --git a/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs b/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs index effd70c7e..d314ce8d7 100644 --- a/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs +++ b/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs @@ -10,7 +10,7 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 2048, n_ubatch = 512, )] -fn logit_bias_with_empty_biases_succeeds(_fixture: &LlamaFixture<'_>) { +fn a_logit_bias_sampler_accepts_an_empty_bias_list(_fixture: &LlamaFixture<'_>) { let result = LlamaSampler::logit_bias(0, &[]); assert!(result.is_ok()); diff --git a/llama-cpp-bindings-tests/tests/logits_retrieval.rs b/llama-cpp-bindings-tests/tests/logits_retrieval.rs index 2c54977b1..22ec950cb 100644 --- a/llama-cpp-bindings-tests/tests/logits_retrieval.rs +++ b/llama-cpp-bindings-tests/tests/logits_retrieval.rs @@ -18,7 +18,7 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 2048, n_ubatch = 512, )] -fn get_logits_ith_returns_valid_slice(fixture: &LlamaFixture<'_>) -> Result<()> { +fn a_logits_row_holds_one_logit_per_vocabulary_token(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = LlamaContext::from_model( fixture.model, fixture.backend, diff --git a/llama-cpp-bindings-tests/tests/model_hyperparameters.rs b/llama-cpp-bindings-tests/tests/model_hyperparameters.rs index 405c23134..8e9b23da0 100644 --- a/llama-cpp-bindings-tests/tests/model_hyperparameters.rs +++ b/llama-cpp-bindings-tests/tests/model_hyperparameters.rs @@ -42,153 +42,10 @@ fn model_reports_positive_core_hyperparameters(fixture: &LlamaFixture<'_>) -> Re assert!(model.n_embd() > 0); assert!(model.n_params() > 0); assert!(model.n_ctx_train()? > 0); + assert!(model.n_layer()? > 0); + assert!(model.n_head()? > 0); + assert!(model.n_head_kv()? > 0); + assert!(model.size() > 0); Ok(()) } - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn n_layer_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(fixture.model.n_layer()? > 0); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn n_head_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(fixture.model.n_head()? > 0); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn n_head_kv_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(fixture.model.n_head_kv()? > 0); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn model_size_returns_nonzero(fixture: &LlamaFixture<'_>) { - assert!(fixture.model.size() > 0); -} diff --git a/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs b/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs index 4ddbc5411..ddf9f2421 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs @@ -27,7 +27,7 @@ use llama_cpp_test_harness_macros::llama_test; n_ubatch = 64, mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), )] -fn decode_use_non_causal_returns_bool_for_image_chunk(fixture: &LlamaFixture<'_>) -> Result<()> { +fn qwen_image_chunks_decode_causally(fixture: &LlamaFixture<'_>) -> Result<()> { let mtmd_ctx = fixture .mtmd_context .expect("mmproj_file declared in attribute"); @@ -44,12 +44,8 @@ fn decode_use_non_causal_returns_bool_for_image_chunk(fixture: &LlamaFixture<'_> .get(chunk_index) .context(format!("missing chunk at index {chunk_index}"))?; if chunk.chunk_type() == Ok(MtmdInputChunkType::Image) { - let value = mtmd_ctx.decode_use_non_causal(&chunk); - let printed = format!("{value:?}"); - assert!( - !printed.is_empty(), - "decode_use_non_causal must return a Debug-printable bool" - ); + assert!(!mtmd_ctx.decode_use_non_causal(&chunk)); + return Ok(()); } } diff --git a/llama-cpp-bindings-tests/tests/parameter_fitting.rs b/llama-cpp-bindings-tests/tests/parameter_fitting.rs index 7a7221e5d..66de725f0 100644 --- a/llama-cpp-bindings-tests/tests/parameter_fitting.rs +++ b/llama-cpp-bindings-tests/tests/parameter_fitting.rs @@ -46,7 +46,7 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 128, n_ubatch = 64, )] -fn fit_params_succeeds_with_test_model(fixture: &LlamaFixture<'_>) -> Result<()> { +fn fit_params_fits_a_nonzero_context_for_the_model(fixture: &LlamaFixture<'_>) -> Result<()> { let model_path_c = CString::new(fixture.model_path.as_os_str().as_encoded_bytes())?; let mut params = pin!(LlamaModelParams::default()); diff --git a/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs b/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs index 8fe82791d..da73b05f6 100644 --- a/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs +++ b/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs @@ -2,7 +2,6 @@ use anyhow::Context as _; use anyhow::Result; use llama_cpp_bindings::sampling::LlamaSampler; -use llama_cpp_bindings::token::LlamaToken; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -14,7 +13,7 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 2048, n_ubatch = 512, )] -fn accept_many_iterates_over_borrowed_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { +fn a_greedy_chain_accepts_many_borrowed_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { let mut sampler = LlamaSampler::chain([LlamaSampler::greedy()?])?; let tokens = vec![ fixture @@ -40,7 +39,7 @@ fn accept_many_iterates_over_borrowed_tokens(fixture: &LlamaFixture<'_>) -> Resu n_batch = 2048, n_ubatch = 512, )] -fn accept_consumes_a_single_token(fixture: &LlamaFixture<'_>) -> Result<()> { +fn a_greedy_chain_accepts_the_bos_token(fixture: &LlamaFixture<'_>) -> Result<()> { let mut sampler = LlamaSampler::chain([LlamaSampler::greedy()?])?; sampler.accept( @@ -52,19 +51,3 @@ fn accept_consumes_a_single_token(fixture: &LlamaFixture<'_>) -> Result<()> { Ok(()) } - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn accept_succeeds_for_a_valid_token(_fixture: &LlamaFixture<'_>) -> Result<()> { - let mut sampler = LlamaSampler::chain([LlamaSampler::greedy()?])?; - - sampler.accept(LlamaToken::new(0))?; - - Ok(()) -} diff --git a/llama-cpp-bindings-tests/tests/session_files.rs b/llama-cpp-bindings-tests/tests/session_files.rs index d9f1982fe..7aedd84b6 100644 --- a/llama-cpp-bindings-tests/tests/session_files.rs +++ b/llama-cpp-bindings-tests/tests/session_files.rs @@ -42,7 +42,9 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 512, n_ubatch = 128, )] -fn state_save_and_load_file_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> { +fn a_loaded_state_file_restores_the_tokens_it_was_saved_with( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let mut context = fixture.build_context()?; let tokens = @@ -96,7 +98,9 @@ fn state_save_and_load_file_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> n_batch = 512, n_ubatch = 128, )] -fn state_seq_save_and_load_file_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> { +fn a_loaded_sequence_state_file_restores_its_tokens_and_bytes( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let mut context = fixture.build_context()?; let tokens = diff --git a/llama-cpp-bindings-tests/tests/state_snapshots.rs b/llama-cpp-bindings-tests/tests/state_snapshots.rs index 9b9cee4f4..ee960e690 100644 --- a/llama-cpp-bindings-tests/tests/state_snapshots.rs +++ b/llama-cpp-bindings-tests/tests/state_snapshots.rs @@ -227,7 +227,9 @@ fn state_seq_get_size_ext_returns_size_for_decoded_sequence( n_batch = 512, n_ubatch = 128, )] -fn state_seq_get_data_ext_and_set_data_ext_round_trip(fixture: &LlamaFixture<'_>) -> Result<()> { +fn state_seq_set_data_ext_consumes_exactly_the_bytes_state_seq_get_data_ext_wrote( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let mut context = fixture.build_context()?; let tokens = diff --git a/llama-cpp-bindings-tests/tests/threadpool_detachment.rs b/llama-cpp-bindings-tests/tests/threadpool_detachment.rs index ab4c5b7dd..95f96020a 100644 --- a/llama-cpp-bindings-tests/tests/threadpool_detachment.rs +++ b/llama-cpp-bindings-tests/tests/threadpool_detachment.rs @@ -12,7 +12,7 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 2048, n_ubatch = 512, )] -fn detach_threadpool_completes_without_panic(fixture: &LlamaFixture<'_>) -> Result<()> { +fn a_context_detaches_its_threadpool(fixture: &LlamaFixture<'_>) -> Result<()> { let context = LlamaContext::from_model( fixture.model, fixture.backend, diff --git a/llama-cpp-bindings-tests/tests/token_candidates.rs b/llama-cpp-bindings-tests/tests/token_candidates.rs index c0d846a38..0a900c3a5 100644 --- a/llama-cpp-bindings-tests/tests/token_candidates.rs +++ b/llama-cpp-bindings-tests/tests/token_candidates.rs @@ -19,7 +19,9 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 2048, n_ubatch = 512, )] -fn token_data_array_ith_returns_valid_data(fixture: &LlamaFixture<'_>) -> Result<()> { +fn a_token_data_array_holds_one_candidate_per_vocabulary_token( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let mut context = LlamaContext::from_model( fixture.model, fixture.backend, diff --git a/llama-cpp-bindings-tests/tests/tokenization.rs b/llama-cpp-bindings-tests/tests/tokenization.rs index 038073dbf..36a6ba08f 100644 --- a/llama-cpp-bindings-tests/tests/tokenization.rs +++ b/llama-cpp-bindings-tests/tests/tokenization.rs @@ -41,7 +41,9 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 128, n_ubatch = 64, )] -fn str_to_token_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> { +fn the_first_token_of_tokenized_text_decodes_to_a_nonempty_piece( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let model = fixture.model; let tokens = model.str_to_token("hello world", AddBos::Never, ParseSpecialTokens::Always)?; assert_ne!(tokens, Vec::::new()); diff --git a/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs b/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs index 5d11a0b8a..fcae981fb 100644 --- a/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs +++ b/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs @@ -12,61 +12,6 @@ use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn sample_returns_result_and_succeeds_with_valid_index(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut context = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let tokens = model.str_to_token("Hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - - batch.add_sequence(&tokens, 0, false)?; - - context.decode(&batch)?; - - let mut sampler = LlamaSampler::chain([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; - - let result = sampler.sample(&context, batch.n_tokens() - 1); - - assert!(result.is_ok()); - Ok(()) -} - #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -163,7 +108,7 @@ fn sample_without_grammar_produces_multiple_tokens(fixture: &LlamaFixture<'_>) - n_batch = 2048, n_ubatch = 512, )] -fn apply_runs_sampler_over_token_data_array(fixture: &LlamaFixture<'_>) -> Result<()> { +fn a_greedy_sampler_selects_a_candidate_of_a_decoded_row(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = LlamaContext::from_model( fixture.model, fixture.backend, @@ -180,6 +125,8 @@ fn apply_runs_sampler_over_token_data_array(fixture: &LlamaFixture<'_>) -> Resul let sampler = LlamaSampler::greedy()?; sampler.apply(&mut data_array)?; + assert_ne!(data_array.selected_token(), None); + Ok(()) } @@ -191,7 +138,7 @@ fn apply_runs_sampler_over_token_data_array(fixture: &LlamaFixture<'_>) -> Resul n_batch = 2048, n_ubatch = 512, )] -fn sample_returns_token_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { +fn sampling_the_last_decoded_row_yields_a_token(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = LlamaContext::from_model( fixture.model, fixture.backend, diff --git a/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs b/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs index 7009d260a..f00da1b47 100644 --- a/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs +++ b/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs @@ -35,7 +35,7 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 128, n_ubatch = 64, )] -fn tokens_iterator_produces_valid_entries(fixture: &LlamaFixture<'_>) { +fn the_vocabulary_iterator_yields_non_negative_token_ids(fixture: &LlamaFixture<'_>) { let model = fixture.model; let mut count = 0; diff --git a/llama-cpp-bindings/src/chat_output.rs b/llama-cpp-bindings/src/chat_output.rs index b6e8476a8..60ddaba2f 100644 --- a/llama-cpp-bindings/src/chat_output.rs +++ b/llama-cpp-bindings/src/chat_output.rs @@ -1,14 +1,14 @@ +use std::ops::Range; + use llama_cpp_bindings_sys::LLAMA_TOKEN_NULL; use llama_cpp_bindings_sys::llama_token; -use crate::chat_output_token::ChatOutputToken; use crate::token::LlamaToken; #[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct ChatOutput { text: String, byte_tokens: Vec, - tokens: Vec, } impl From<&str> for ChatOutput { @@ -17,15 +17,15 @@ impl From<&str> for ChatOutput { Self { text: text.to_owned(), byte_tokens: vec![LLAMA_TOKEN_NULL; text.len()], - tokens: Vec::new(), } } } impl ChatOutput { /// Appends the text `token` decoded to, recording the token at the first byte of its piece - /// the way `common_chat_input` lays tokens over the text. - pub fn push(&mut self, token: LlamaToken, piece: &str) { + /// the way `common_chat_input` lays tokens over the text, and returns the bytes the piece + /// occupies. + pub fn push(&mut self, token: LlamaToken, piece: &str) -> Range { let start = self.text.len(); self.text.push_str(piece); @@ -36,10 +36,8 @@ impl ChatOutput { LLAMA_TOKEN_NULL } })); - self.tokens.push(ChatOutputToken { - token, - bytes: start..self.text.len(), - }); + + start..self.text.len() } #[must_use] @@ -51,11 +49,6 @@ impl ChatOutput { pub fn byte_tokens(&self) -> &[llama_token] { &self.byte_tokens } - - #[must_use] - pub fn tokens(&self) -> &[ChatOutputToken] { - &self.tokens - } } #[cfg(test)] @@ -63,46 +56,30 @@ mod tests { use llama_cpp_bindings_sys::LLAMA_TOKEN_NULL; use crate::chat_output::ChatOutput; - use crate::chat_output_token::ChatOutputToken; use crate::token::LlamaToken; #[test] fn each_token_owns_the_bytes_of_its_piece_and_marks_its_first_byte() { let mut output = ChatOutput::default(); - output.push(LlamaToken::new(7), ""); - output.push(LlamaToken::new(9), ""); - output.push(LlamaToken::new(11), "hi"); + let pieces = [ + output.push(LlamaToken::new(7), ""), + output.push(LlamaToken::new(9), ""), + output.push(LlamaToken::new(11), "hi"), + ]; + assert_eq!(pieces, [0..3, 3..3, 3..5]); assert_eq!(output.text(), "hi"); assert_eq!( output.byte_tokens(), [7, LLAMA_TOKEN_NULL, LLAMA_TOKEN_NULL, 11, LLAMA_TOKEN_NULL] ); - assert_eq!( - output.tokens(), - [ - ChatOutputToken { - token: LlamaToken::new(7), - bytes: 0..3, - }, - ChatOutputToken { - token: LlamaToken::new(9), - bytes: 3..3, - }, - ChatOutputToken { - token: LlamaToken::new(11), - bytes: 3..5, - }, - ] - ); } #[test] - fn plain_text_carries_no_tokens() { + fn plain_text_marks_no_byte_with_a_token() { let output = ChatOutput::from("ab"); assert_eq!(output.byte_tokens(), [LLAMA_TOKEN_NULL, LLAMA_TOKEN_NULL]); - assert_eq!(output.tokens(), []); } } diff --git a/llama-cpp-bindings/src/chat_session.rs b/llama-cpp-bindings/src/chat_session.rs index 7d0213f1c..e768b1bcf 100644 --- a/llama-cpp-bindings/src/chat_session.rs +++ b/llama-cpp-bindings/src/chat_session.rs @@ -257,11 +257,7 @@ fn read_chat_spans( }) }) .collect::, FfiStatusError>>() - .map(|spans_read| ChatSpans { - parsed_length: view.parsed_length, - spans: spans_read, - completeness, - }) + .map(|spans_read| ChatSpans::new(view.parsed_length, &spans_read, completeness)) .map_err(ParseChatMessageError::from); unsafe { llama_cpp_bindings_sys::llama_rs_chat_spans_free(spans.as_ptr()) }; diff --git a/llama-cpp-bindings/src/chat_span_segment.rs b/llama-cpp-bindings/src/chat_span_segment.rs new file mode 100644 index 000000000..3732a42eb --- /dev/null +++ b/llama-cpp-bindings/src/chat_span_segment.rs @@ -0,0 +1,9 @@ +use std::ops::Range; + +use crate::chat_piece_role::ChatPieceRole; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ChatSpanSegment { + pub bytes: Range, + pub role: ChatPieceRole, +} diff --git a/llama-cpp-bindings/src/chat_spans.rs b/llama-cpp-bindings/src/chat_spans.rs index 22aaa04e9..36cacfa75 100644 --- a/llama-cpp-bindings/src/chat_spans.rs +++ b/llama-cpp-bindings/src/chat_spans.rs @@ -4,38 +4,91 @@ use std::ops::Range; use crate::chat_output_completeness::ChatOutputCompleteness; use crate::chat_piece_role::ChatPieceRole; use crate::chat_span::ChatSpan; +use crate::chat_span_segment::ChatSpanSegment; use crate::sampled_token_section::SampledTokenSection; +fn innermost_role_at(spans: &[ChatSpan], byte: usize) -> Option { + spans + .iter() + .enumerate() + .filter(|(_order, span)| span.bytes.contains(&byte)) + .min_by_key(|(order, span)| (span.bytes.len(), Reverse(*order))) + .map(|(_order, span)| span.kind.piece_role()) +} + +fn segments_of(spans: &[ChatSpan]) -> Vec { + let mut boundaries = spans + .iter() + .flat_map(|span| [span.bytes.start, span.bytes.end]) + .collect::>(); + + boundaries.sort_unstable(); + boundaries.dedup(); + + boundaries + .iter() + .zip(boundaries.iter().skip(1)) + .filter_map(|(&start, &end)| { + innermost_role_at(spans, start).map(|role| ChatSpanSegment { + bytes: start..end, + role, + }) + }) + .collect() +} + #[derive(Clone, Debug, Eq, PartialEq)] pub struct ChatSpans { - pub parsed_length: usize, - pub spans: Vec, - pub completeness: ChatOutputCompleteness, + parsed_length: usize, + segments: Vec, + completeness: ChatOutputCompleteness, } impl ChatSpans { + /// Splits the nested `spans` of llama.cpp's parse once into ordered segments, each carrying + /// the role of the innermost span over it, so roles are looked up by binary search. + #[must_use] + pub fn new( + parsed_length: usize, + spans: &[ChatSpan], + completeness: ChatOutputCompleteness, + ) -> Self { + Self { + parsed_length, + segments: segments_of(spans), + completeness, + } + } + + fn first_segment_ending_after(&self, byte: usize) -> usize { + self.segments + .partition_point(|segment| segment.bytes.end <= byte) + } + fn tagged_role_at(&self, byte: usize) -> Option { - self.spans - .iter() - .enumerate() - .filter(|(_order, span)| span.bytes.contains(&byte)) - .min_by_key(|(order, span)| (span.bytes.len(), Reverse(*order))) - .map(|(_order, span)| span.kind.piece_role()) + self.segments + .get(self.first_segment_ending_after(byte)) + .filter(|segment| segment.bytes.contains(&byte)) + .map(|segment| segment.role) } fn untagged_marker_section(&self, byte: usize) -> Option { - let next = (byte..self.parsed_length) - .find_map(|later| self.tagged_role_at(later)) - .map(ChatPieceRole::section); + let next = self + .segments + .get(self.first_segment_ending_after(byte)) + .filter(|segment| segment.bytes.start.max(byte) < self.parsed_length) + .map(|segment| segment.role.section()); if next.is_none() && self.completeness == ChatOutputCompleteness::Partial { return None; } - let previous = (0..byte) - .rev() - .find_map(|earlier| self.tagged_role_at(earlier)) - .map(ChatPieceRole::section); + let previous = self + .segments + .partition_point(|segment| segment.bytes.start < byte) + .checked_sub(1) + .and_then(|index| self.segments.get(index)) + .map(|segment| segment.role.section()); Some(match (previous, next) { (Some(SampledTokenSection::Reasoning), _) @@ -55,9 +108,11 @@ impl ChatSpans { return None; } - bytes - .clone() - .filter_map(|byte| self.tagged_role_at(byte)) + self.segments + .iter() + .skip(self.first_segment_ending_after(bytes.start)) + .take_while(|segment| segment.bytes.start < bytes.end) + .map(|segment| segment.role) .find(|role| matches!(role, ChatPieceRole::Text(_))) .or_else(|| { bytes @@ -89,14 +144,14 @@ mod tests { } fn reasoning_then_content(completeness: ChatOutputCompleteness) -> ChatSpans { - ChatSpans { - parsed_length: 30, - spans: vec![ + ChatSpans::new( + 30, + &[ span(ChatSpanKind::Reasoning, 7, 12), span(ChatSpanKind::Content, 22, 30), ], completeness, - } + ) } #[test] @@ -141,14 +196,14 @@ mod tests { #[test] fn the_innermost_span_decides_a_byte_inside_nested_spans() { - let spans = ChatSpans { - parsed_length: 20, - spans: vec![ + let spans = ChatSpans::new( + 20, + &[ span(ChatSpanKind::ToolCall, 0, 20), span(ChatSpanKind::ToolCallMarker, 0, 6), ], - completeness: ChatOutputCompleteness::Partial, - }; + ChatOutputCompleteness::Partial, + ); assert_eq!( [ @@ -164,14 +219,14 @@ mod tests { #[test] fn an_untagged_token_beside_a_tool_call_marks_the_tool_call() { - let spans = ChatSpans { - parsed_length: 20, - spans: vec![ + let spans = ChatSpans::new( + 20, + &[ span(ChatSpanKind::Content, 0, 5), span(ChatSpanKind::ToolCall, 10, 20), ], - completeness: ChatOutputCompleteness::Partial, - }; + ChatOutputCompleteness::Partial, + ); assert_eq!( spans.classify_token(&(5..10)), @@ -181,33 +236,33 @@ mod tests { #[test] fn an_untagged_partial_tail_after_content_waits_for_what_follows() { - let spans = ChatSpans { - parsed_length: 8, - spans: vec![span(ChatSpanKind::Content, 0, 5)], - completeness: ChatOutputCompleteness::Partial, - }; + let spans = ChatSpans::new( + 8, + &[span(ChatSpanKind::Content, 0, 5)], + ChatOutputCompleteness::Partial, + ); assert_eq!(spans.classify_token(&(5..8)), None); } #[test] fn an_untagged_partial_tail_after_reasoning_waits_for_what_follows() { - let spans = ChatSpans { - parsed_length: 13, - spans: vec![span(ChatSpanKind::Reasoning, 0, 12)], - completeness: ChatOutputCompleteness::Partial, - }; + let spans = ChatSpans::new( + 13, + &[span(ChatSpanKind::Reasoning, 0, 12)], + ChatOutputCompleteness::Partial, + ); assert_eq!(spans.classify_token(&(12..13)), None); } #[test] fn an_untagged_tail_of_a_finished_parse_after_content_is_a_content_marker() { - let spans = ChatSpans { - parsed_length: 8, - spans: vec![span(ChatSpanKind::Content, 0, 5)], - completeness: ChatOutputCompleteness::Finished, - }; + let spans = ChatSpans::new( + 8, + &[span(ChatSpanKind::Content, 0, 5)], + ChatOutputCompleteness::Finished, + ); assert_eq!( spans.classify_token(&(5..8)), @@ -225,14 +280,14 @@ mod tests { #[test] fn a_reasoning_block_byte_outside_reasoning_text_is_a_reasoning_marker() { - let spans = ChatSpans { - parsed_length: 20, - spans: vec![ + let spans = ChatSpans::new( + 20, + &[ span(ChatSpanKind::ReasoningBlock, 0, 20), span(ChatSpanKind::Reasoning, 7, 12), ], - completeness: ChatOutputCompleteness::Partial, - }; + ChatOutputCompleteness::Partial, + ); assert_eq!( spans.classify_token(&(12..20)), diff --git a/llama-cpp-bindings/src/context/llama_attention_type.rs b/llama-cpp-bindings/src/context/llama_attention_type.rs index 79b9f66f6..3afc344fc 100644 --- a/llama-cpp-bindings/src/context/llama_attention_type.rs +++ b/llama-cpp-bindings/src/context/llama_attention_type.rs @@ -43,7 +43,7 @@ mod tests { } #[test] - fn attention_type_roundtrip_all_variants() { + fn every_attention_type_converts_to_llama_cpp_and_back() { for (raw, expected) in [ (-1, LlamaAttentionType::Unspecified), (0, LlamaAttentionType::Causal), diff --git a/llama-cpp-bindings/src/context/llama_pooling_type.rs b/llama-cpp-bindings/src/context/llama_pooling_type.rs index 651216f39..38f9c423e 100644 --- a/llama-cpp-bindings/src/context/llama_pooling_type.rs +++ b/llama-cpp-bindings/src/context/llama_pooling_type.rs @@ -46,7 +46,7 @@ mod tests { } #[test] - fn pooling_type_roundtrip_all_variants() { + fn every_pooling_type_converts_to_llama_cpp_and_back() { for (raw, expected) in [ (-1, LlamaPoolingType::Unspecified), (0, LlamaPoolingType::None), diff --git a/llama-cpp-bindings/src/context/params.rs b/llama-cpp-bindings/src/context/params.rs index 264ca3162..0b6725bfd 100644 --- a/llama-cpp-bindings/src/context/params.rs +++ b/llama-cpp-bindings/src/context/params.rs @@ -341,7 +341,7 @@ mod tests { use super::RopeScalingType; #[test] - fn default_params_have_expected_values() { + fn default_params_match_the_llama_cpp_context_defaults() { let params = super::LlamaContextParams::default(); assert_eq!(params.n_ctx(), NonZeroU32::new(512)); diff --git a/llama-cpp-bindings/src/context/rope_scaling_type.rs b/llama-cpp-bindings/src/context/rope_scaling_type.rs index 92aca372e..8a0ea0577 100644 --- a/llama-cpp-bindings/src/context/rope_scaling_type.rs +++ b/llama-cpp-bindings/src/context/rope_scaling_type.rs @@ -40,7 +40,7 @@ mod tests { } #[test] - fn rope_scaling_type_roundtrip_all_variants() { + fn every_rope_scaling_type_converts_to_llama_cpp_and_back() { for (raw, expected) in [ (-1, RopeScalingType::Unspecified), (0, RopeScalingType::None), diff --git a/llama-cpp-bindings/src/gguf_type.rs b/llama-cpp-bindings/src/gguf_type.rs index e59451e12..31b3fe054 100644 --- a/llama-cpp-bindings/src/gguf_type.rs +++ b/llama-cpp-bindings/src/gguf_type.rs @@ -74,7 +74,7 @@ mod tests { } #[test] - fn to_raw_roundtrips() { + fn every_known_gguf_type_converts_back_to_its_raw_value() { for raw in 0..=12 { let gguf_type = GgufType::from_raw(raw).unwrap(); assert_eq!(gguf_type.to_raw(), raw); diff --git a/llama-cpp-bindings/src/lib.rs b/llama-cpp-bindings/src/lib.rs index 8516d2f25..f7814c848 100644 --- a/llama-cpp-bindings/src/lib.rs +++ b/llama-cpp-bindings/src/lib.rs @@ -14,13 +14,13 @@ pub mod chat_grammar_trigger; pub mod chat_message_parse_outcome; pub mod chat_output; pub mod chat_output_completeness; -pub mod chat_output_token; pub mod chat_piece_role; pub mod chat_prompt_part; pub mod chat_session; pub mod chat_session_options; pub mod chat_span; pub mod chat_span_kind; +mod chat_span_segment; pub mod chat_spans; pub mod chat_template_tool_calls; pub mod chat_templates; @@ -65,6 +65,7 @@ pub mod max_devices; pub mod model; pub mod mtmd; pub mod parsed_chat; +mod pending_chat_piece; pub mod raw_chat_message; pub mod sampled_token; pub mod sampled_token_classifier; diff --git a/llama-cpp-bindings/src/llama_backend.rs b/llama-cpp-bindings/src/llama_backend.rs index 0b6d6b982..ebaa4e7d6 100644 --- a/llama-cpp-bindings/src/llama_backend.rs +++ b/llama-cpp-bindings/src/llama_backend.rs @@ -74,13 +74,14 @@ mod tests { use super::LlamaBackend; use crate::error::llama_cpp_error::LlamaCppError; + use crate::llama_backend_device::LlamaBackendDevice; + use crate::llama_backend_device_type::LlamaBackendDeviceType; use crate::llama_backend_numa_strategy::NumaStrategy; #[test] #[serial] - fn init_succeeds() { - let backend = LlamaBackend::init(); - assert!(backend.is_ok()); + fn initializing_the_backend_returns_it() { + assert_eq!(LlamaBackend::init(), Ok(LlamaBackend {})); } #[test] @@ -111,11 +112,28 @@ mod tests { #[test] #[serial] - fn feature_queries_return_bools() { + fn the_backend_supports_mmap_and_mlock() { let backend = LlamaBackend::init().unwrap(); - let _gpu = backend.supports_gpu_offload(); - let _mmap = backend.supports_mmap(); - let _mlock = backend.supports_mlock(); + + assert_eq!( + [backend.supports_mmap(), backend.supports_mlock()], + [true, true] + ); + } + + #[test] + #[serial] + fn the_backend_offloads_to_a_gpu_exactly_when_one_is_registered() { + let backend = LlamaBackend::init().unwrap(); + let offloads_to_a_gpu = backend.supports_gpu_offload(); + + assert_eq!( + offloads_to_a_gpu, + LlamaBackendDevice::list_all().iter().any(|device| matches!( + device.device_type, + LlamaBackendDeviceType::Gpu | LlamaBackendDeviceType::IntegratedGpu + )) + ); } #[test] diff --git a/llama-cpp-bindings/src/llama_backend_device.rs b/llama-cpp-bindings/src/llama_backend_device.rs index 3b9eba3f5..e56348a2c 100644 --- a/llama-cpp-bindings/src/llama_backend_device.rs +++ b/llama-cpp-bindings/src/llama_backend_device.rs @@ -77,7 +77,7 @@ mod tests { } #[test] - fn cstr_to_string_with_valid_ptr() { + fn cstr_to_string_copies_a_non_null_string() { let result = cstr_to_string(c"hello".as_ptr()); assert_eq!(result, "hello"); diff --git a/llama-cpp-bindings/src/llama_backend_numa_strategy.rs b/llama-cpp-bindings/src/llama_backend_numa_strategy.rs index fccef05aa..b0044917c 100644 --- a/llama-cpp-bindings/src/llama_backend_numa_strategy.rs +++ b/llama-cpp-bindings/src/llama_backend_numa_strategy.rs @@ -42,7 +42,7 @@ mod tests { use crate::invalid_numa_strategy::InvalidNumaStrategy; #[test] - fn numa_from_and_to() { + fn every_numa_strategy_converts_to_ggml_and_back() { let numas = [ NumaStrategy::Disabled, NumaStrategy::Distribute, diff --git a/llama-cpp-bindings/src/llama_time_us.rs b/llama-cpp-bindings/src/llama_time_us.rs index 1a5626fc7..1344a4783 100644 --- a/llama-cpp-bindings/src/llama_time_us.rs +++ b/llama-cpp-bindings/src/llama_time_us.rs @@ -8,7 +8,7 @@ mod tests { use crate::llama_time_us::llama_time_us; #[test] - fn returns_positive_value() { + fn llama_time_is_positive() { let time_microseconds = llama_time_us(); assert!(time_microseconds > 0); diff --git a/llama-cpp-bindings/src/max_devices.rs b/llama-cpp-bindings/src/max_devices.rs index 910225d59..1d2058bc3 100644 --- a/llama-cpp-bindings/src/max_devices.rs +++ b/llama-cpp-bindings/src/max_devices.rs @@ -8,7 +8,7 @@ mod tests { use crate::max_devices::max_devices; #[test] - fn returns_at_least_one() { + fn at_least_one_device_is_supported() { let device_count = max_devices(); assert!(device_count >= 1); diff --git a/llama-cpp-bindings/src/model/llama_chat_message.rs b/llama-cpp-bindings/src/model/llama_chat_message.rs index b64b91df3..3844b34fc 100644 --- a/llama-cpp-bindings/src/model/llama_chat_message.rs +++ b/llama-cpp-bindings/src/model/llama_chat_message.rs @@ -45,7 +45,7 @@ mod tests { } #[test] - fn empty_strings_are_valid() { + fn an_empty_role_and_content_form_a_message() { let message = LlamaChatMessage::new(String::new(), String::new()); assert!(message.is_ok()); diff --git a/llama-cpp-bindings/src/model/llama_chat_template.rs b/llama-cpp-bindings/src/model/llama_chat_template.rs index ac4f82337..3ef57e392 100644 --- a/llama-cpp-bindings/src/model/llama_chat_template.rs +++ b/llama-cpp-bindings/src/model/llama_chat_template.rs @@ -44,7 +44,7 @@ mod tests { use crate::model::llama_chat_template::LlamaChatTemplate; #[test] - fn valid_template_creation() { + fn a_template_keeps_its_name() { let template = LlamaChatTemplate::new("chatml").unwrap(); let template_str = template.to_str().unwrap(); @@ -67,7 +67,7 @@ mod tests { } #[test] - fn to_string_returns_owned_string() { + fn to_string_copies_the_template_name() { let template = LlamaChatTemplate::new("llama3").unwrap(); let owned = template.to_string().unwrap(); @@ -75,7 +75,7 @@ mod tests { } #[test] - fn as_c_str_returns_valid_cstr() { + fn as_c_str_views_the_template_name() { let template = LlamaChatTemplate::new("test").unwrap(); let cstr = template.as_c_str(); diff --git a/llama-cpp-bindings/src/model/params.rs b/llama-cpp-bindings/src/model/params.rs index 586b806ec..2194d15b0 100644 --- a/llama-cpp-bindings/src/model/params.rs +++ b/llama-cpp-bindings/src/model/params.rs @@ -537,7 +537,7 @@ mod tests { } #[test] - fn default_params_have_expected_values() { + fn default_params_match_the_llama_cpp_model_defaults() { let params = LlamaModelParams::default(); assert_eq!(params.n_gpu_layers(), -1); @@ -585,13 +585,6 @@ mod tests { assert_eq!(params.n_gpu_layers(), 999); } - #[test] - fn with_n_gpu_layers_sets_value() { - let params = LlamaModelParams::default().with_n_gpu_layers(32); - - assert_eq!(params.n_gpu_layers(), 32); - } - #[test] fn with_main_gpu_sets_value() { let params = LlamaModelParams::default().with_main_gpu(2); diff --git a/llama-cpp-bindings/src/model/params/kv_overrides.rs b/llama-cpp-bindings/src/model/params/kv_overrides.rs index 0d8888876..b27ea6083 100644 --- a/llama-cpp-bindings/src/model/params/kv_overrides.rs +++ b/llama-cpp-bindings/src/model/params/kv_overrides.rs @@ -65,15 +65,6 @@ mod tests { ); } - #[test] - fn kv_overrides_new_creates_view() { - let params = LlamaModelParams::default(); - let overrides = super::KvOverrides::new(¶ms); - let count = overrides.into_iter().count(); - - assert_eq!(count, 0); - } - #[test] fn kv_overrides_preserves_unknown_tag_error() { let mut params = pin!(LlamaModelParams::default()); diff --git a/llama-cpp-bindings/src/model/params/param_override_value.rs b/llama-cpp-bindings/src/model/params/param_override_value.rs index 8bcbb87e5..f8c18a290 100644 --- a/llama-cpp-bindings/src/model/params/param_override_value.rs +++ b/llama-cpp-bindings/src/model/params/param_override_value.rs @@ -121,7 +121,7 @@ mod tests { } #[test] - fn from_ffi_override_int() { + fn an_int_kv_override_reads_as_an_int_value() { let ffi_override = llama_cpp_bindings_sys::llama_model_kv_override { key: [0; 128], tag: llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_INT, @@ -136,7 +136,7 @@ mod tests { } #[test] - fn from_ffi_override_float() { + fn a_float_kv_override_reads_as_a_float_value() { let ffi_override = llama_cpp_bindings_sys::llama_model_kv_override { key: [0; 128], tag: llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_FLOAT, @@ -151,7 +151,7 @@ mod tests { } #[test] - fn from_ffi_override_bool() { + fn a_bool_kv_override_reads_as_a_bool_value() { let ffi_override = llama_cpp_bindings_sys::llama_model_kv_override { key: [0; 128], tag: llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_BOOL, @@ -180,7 +180,7 @@ mod tests { } #[test] - fn from_ffi_override_str() { + fn a_str_kv_override_reads_as_a_str_value() { let mut str_data = [0i8; 128]; str_data[0] = b'a'.cast_signed(); str_data[1] = b'b'.cast_signed(); diff --git a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs index fe99face6..4951e89b7 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs @@ -414,7 +414,7 @@ mod tests { use crate::mtmd::mtmd_bitmap_error::MtmdBitmapError; #[test] - fn from_image_data_creates_valid_bitmap() { + fn image_data_makes_an_image_bitmap_of_its_dimensions() { let red_pixel: [u8; 3] = [255, 0, 0]; let image_data: Vec = red_pixel.repeat(4); let bitmap = MtmdBitmap::from_image_data(2, 2, &image_data).unwrap(); @@ -479,7 +479,7 @@ mod tests { } #[test] - fn from_audio_data_creates_valid_bitmap() { + fn audio_samples_make_an_audio_bitmap() { let audio_samples: Vec = (0u8..100) .map(|index| (f32::from(index) * 0.1).sin()) .collect(); @@ -537,7 +537,7 @@ mod cstr_ptr_to_optional_string_tests { } #[test] - fn cstr_ptr_to_optional_string_returns_some_for_valid() { + fn cstr_ptr_to_optional_string_copies_a_non_null_string() { let cstr = CString::new("hello").unwrap(); let result = cstr_ptr_to_optional_string(cstr.as_ptr()); diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs index d5685bb60..25b189efa 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs @@ -227,6 +227,7 @@ mod unit_tests { use super::tokens_from_raw_ptr; use crate::mtmd::mtmd_eval_error::MtmdEvalError; use crate::mtmd::non_causal_chunk_micro_batch_mismatch::NonCausalChunkMicroBatchMismatch; + use crate::token::LlamaToken; #[test] fn tokens_from_raw_ptr_returns_none_for_null() { @@ -240,12 +241,11 @@ mod unit_tests { } #[test] - fn tokens_from_raw_ptr_returns_some_for_valid() { + fn tokens_from_raw_ptr_views_every_token() { let tokens: [llama_cpp_bindings_sys::llama_token; 2] = [1, 2]; let result = unsafe { tokens_from_raw_ptr(tokens.as_ptr(), 2) }; - assert!(result.is_some()); - assert_eq!(result.unwrap().len(), 2); + assert_eq!(result, Some(&[LlamaToken::new(1), LlamaToken::new(2)][..])); } #[test] diff --git a/llama-cpp-bindings/src/chat_output_token.rs b/llama-cpp-bindings/src/pending_chat_piece.rs similarity index 61% rename from llama-cpp-bindings/src/chat_output_token.rs rename to llama-cpp-bindings/src/pending_chat_piece.rs index 6e30f52d4..d5e3219a0 100644 --- a/llama-cpp-bindings/src/chat_output_token.rs +++ b/llama-cpp-bindings/src/pending_chat_piece.rs @@ -2,8 +2,9 @@ use std::ops::Range; use crate::token::LlamaToken; -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct ChatOutputToken { +#[derive(Debug)] +pub struct PendingChatPiece { pub token: LlamaToken, pub bytes: Range, + pub piece: String, } diff --git a/llama-cpp-bindings/src/sampled_token_classifier.rs b/llama-cpp-bindings/src/sampled_token_classifier.rs index e820f6890..947157336 100644 --- a/llama-cpp-bindings/src/sampled_token_classifier.rs +++ b/llama-cpp-bindings/src/sampled_token_classifier.rs @@ -1,3 +1,5 @@ +use std::collections::VecDeque; + use llama_cpp_bindings_sys::llama_pos; use llama_cpp_bindings_sys::llama_seq_id; use llama_cpp_bindings_types::token_usage::TokenUsage; @@ -26,6 +28,7 @@ use crate::mtmd::mtmd_context::MtmdContext; use crate::mtmd::mtmd_eval_limits::MtmdEvalLimits; use crate::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; use crate::mtmd::mtmd_input_chunks::MtmdInputChunks; +use crate::pending_chat_piece::PendingChatPiece; use crate::sampled_token::SampledToken; use crate::sampled_token_section::SampledTokenSection; use crate::sampling::LlamaSampler; @@ -57,7 +60,7 @@ struct ChatSessionAttribution<'session, 'model> { session: &'session ChatSession<'model>, preserved_tokens: Vec, output: ChatOutput, - attributed_tokens: usize, + pending_pieces: VecDeque, content_probe: ContentProbe, } @@ -125,7 +128,7 @@ impl<'session, 'model> SampledTokenClassifier<'session, 'model> { session, preserved_tokens: session.preserved_tokens(), output: ChatOutput::default(), - attributed_tokens: 0, + pending_pieces: VecDeque::new(), content_probe, }), decoder: encoding_rs::UTF_8.new_decoder(), @@ -181,7 +184,13 @@ impl<'session, 'model> SampledTokenClassifier<'session, 'model> { None, )?; - attribution.output.push(token, &piece); + let bytes = attribution.output.push(token, &piece); + + attribution.pending_pieces.push_back(PendingChatPiece { + token, + bytes, + piece, + }); self.attribute(ChatOutputCompleteness::Partial, outcomes)?; Ok(GenerationProgress::Continues) @@ -200,20 +209,16 @@ impl<'session, 'model> SampledTokenClassifier<'session, 'model> { .classify(&attribution.output, completeness)?; let mut released = Vec::new(); - while let Some(output_token) = attribution - .output - .tokens() - .get(attribution.attributed_tokens) - { - let Some(role) = classification.classify_token(&output_token.bytes) else { + while let Some(pending_piece) = attribution.pending_pieces.pop_front() { + let Some(role) = classification.classify_token(&pending_piece.bytes) else { + attribution.pending_pieces.push_front(pending_piece); break; }; - let piece = attribution.output.text()[output_token.bytes.clone()].to_owned(); - attribution - .content_probe - .pass(outcome_of(output_token.token, role, piece), &mut released); - attribution.attributed_tokens += 1; + attribution.content_probe.pass( + outcome_of(pending_piece.token, role, pending_piece.piece), + &mut released, + ); } if completeness == ChatOutputCompleteness::Finished { diff --git a/llama-cpp-bindings/src/sampling.rs b/llama-cpp-bindings/src/sampling.rs index 07f18ef6f..e1c18106c 100644 --- a/llama-cpp-bindings/src/sampling.rs +++ b/llama-cpp-bindings/src/sampling.rs @@ -1075,7 +1075,7 @@ mod tests { } #[test] - fn sanitize_grammar_strings_valid() { + fn sanitize_grammar_strings_converts_a_grammar_and_root_without_nul_bytes() { assert_eq!( LlamaSampler::sanitize_grammar_strings("root ::= \"hello\"", "root"), Ok(SanitizedGrammar { @@ -1117,7 +1117,7 @@ mod tests { } #[test] - fn sanitize_trigger_patterns_valid() { + fn sanitize_trigger_patterns_converts_every_pattern() { let patterns = vec!["^hello$".to_string(), "world.*".to_string()]; let result = LlamaSampler::sanitize_trigger_patterns(&patterns); @@ -1149,7 +1149,7 @@ mod tests { } #[test] - fn apply_modifies_data_array() { + fn a_greedy_sampler_selects_the_highest_logit() { let sampler = greedy_sampler(); let mut data_array = LlamaTokenDataArray::new( vec![ @@ -1201,17 +1201,7 @@ mod tests { } #[test] - fn accept_succeeds_on_penalties_sampler() { - let mut sampler = LlamaSampler::chain([penalties_sampler(), greedy_sampler()]) - .expect("sampler chain must initialize"); - - let result = sampler.accept(LlamaToken::new(42)); - - assert!(result.is_ok()); - } - - #[test] - fn accept_many_multiple_tokens() { + fn a_penalties_chain_accepts_many_tokens() { let mut sampler = LlamaSampler::chain([penalties_sampler(), greedy_sampler()]) .expect("sampler chain must initialize"); @@ -1221,7 +1211,7 @@ mod tests { } #[test] - fn all_sampler_constructors() { + fn every_sampler_constructor_initializes_its_sampler() { let _temp = LlamaSampler::temp(0.8).expect("temperature sampler must initialize"); let _temp_ext = LlamaSampler::temp_ext(0.8, 0.1, 1.0) .expect("extended temperature sampler must initialize"); @@ -1244,10 +1234,11 @@ mod tests { } #[test] - fn reset_and_get_seed() { + fn a_distribution_sampler_resets_and_reports_its_seed() { let mut sampler = LlamaSampler::dist(42).expect("distribution sampler must initialize"); - assert!(sampler.reset().is_ok()); - let _seed = sampler.get_seed(); + + assert_eq!(sampler.reset(), Ok(())); + assert_eq!(sampler.get_seed(), 42); } #[test] @@ -1267,7 +1258,7 @@ mod checked_usize_as_i32_sampling_tests { use crate::token::logit_bias::LlamaLogitBias; #[test] - fn checked_usize_as_i32_sampling_overflow() { + fn a_count_beyond_an_i32_is_an_integer_overflow() { assert_eq!( checked_usize_as_i32_sampling(usize::MAX), i32::try_from(usize::MAX).map_err(SamplingError::IntegerOverflow) diff --git a/llama-cpp-bindings/src/send_logs_to_log.rs b/llama-cpp-bindings/src/send_logs_to_log.rs index 82b5f1d09..89cd19980 100644 --- a/llama-cpp-bindings/src/send_logs_to_log.rs +++ b/llama-cpp-bindings/src/send_logs_to_log.rs @@ -275,7 +275,7 @@ mod tests { } #[test] - fn ggml_level_to_incoming_known_constants() { + fn every_ggml_log_level_maps_to_its_incoming_level() { assert_eq!( ggml_level_to_incoming(llama_cpp_bindings_sys::GGML_LOG_LEVEL_NONE), IncomingLogLevel::None, @@ -303,7 +303,7 @@ mod tests { } #[test] - fn ggml_level_to_incoming_unknown_value() { + fn an_unknown_ggml_log_level_keeps_its_raw_value() { assert_eq!( ggml_level_to_incoming(9999), IncomingLogLevel::Unknown(9999) @@ -311,7 +311,7 @@ mod tests { } #[test] - fn dispatch_when_disabled() { + fn disabled_logs_emit_no_record() { ensure_test_logger_installed(); let target = "test-dispatch-when-disabled"; @@ -326,7 +326,7 @@ mod tests { } #[test] - fn demote_info_to_debug_on_info() { + fn demoting_info_logs_emits_info_as_debug() { ensure_test_logger_installed(); let target = "test-demote-info-on-info"; @@ -350,7 +350,7 @@ mod tests { } #[test] - fn demote_info_to_debug_on_warn() { + fn demoting_info_logs_keeps_warnings_as_warnings() { ensure_test_logger_installed(); let target = "test-demote-info-on-warn"; @@ -374,7 +374,7 @@ mod tests { } #[test] - fn dispatch_unknown_level() { + fn an_unknown_level_is_logged_as_a_warning_naming_it() { ensure_test_logger_installed(); let target = "test-dispatch-unknown-level"; @@ -391,7 +391,7 @@ mod tests { } #[test] - fn dispatch_orphan_cont_anomaly() { + fn an_orphan_continuation_is_logged_with_an_anomaly_warning() { ensure_test_logger_installed(); let target = "test-dispatch-orphan-cont"; @@ -482,7 +482,7 @@ mod tests { #[test] #[serial] - fn send_logs_to_log_initialization() { + fn sending_logs_to_log_installs_both_log_sources() { ensure_test_logger_installed(); send_logs_to_log(LogOptions::default()); @@ -491,7 +491,7 @@ mod tests { } #[test] - fn null_text_pointer() { + fn a_null_text_pointer_is_logged_as_a_warning() { ensure_test_logger_installed(); let target = "test-null-text-pointer"; @@ -515,7 +515,7 @@ mod tests { } #[test] - fn decoder_mutex_poison() { + fn a_poisoned_decoder_still_decodes_logs() { ensure_test_logger_installed(); let target = "test-decoder-mutex-poison"; diff --git a/llama-cpp-log-decoder/src/incoming_log_level.rs b/llama-cpp-log-decoder/src/incoming_log_level.rs index 3cce5605c..9dc8cbce0 100644 --- a/llama-cpp-log-decoder/src/incoming_log_level.rs +++ b/llama-cpp-log-decoder/src/incoming_log_level.rs @@ -14,7 +14,7 @@ mod tests { use super::IncomingLogLevel; #[test] - fn unknown_variant_equality() { + fn unknown_levels_are_equal_only_with_the_same_raw_value() { assert_eq!(IncomingLogLevel::Unknown(42), IncomingLogLevel::Unknown(42)); assert_ne!(IncomingLogLevel::Unknown(42), IncomingLogLevel::Unknown(43)); } diff --git a/llama-cpp-log-decoder/src/log_decoder.rs b/llama-cpp-log-decoder/src/log_decoder.rs index 166db2559..0b5284acb 100644 --- a/llama-cpp-log-decoder/src/log_decoder.rs +++ b/llama-cpp-log-decoder/src/log_decoder.rs @@ -163,7 +163,7 @@ mod tests { use crate::log_line::LogLine; #[test] - fn feed_complete_info_line() { + fn a_complete_line_is_decoded_without_its_newline() { let mut decoder = LogDecoder::new(); let result = decoder.feed(IncomingLogLevel::Info, "hello\n"); @@ -180,7 +180,7 @@ mod tests { } #[test] - fn feed_partial_without_newline() { + fn a_line_without_a_newline_is_held_back() { let mut decoder = LogDecoder::new(); let result = decoder.feed(IncomingLogLevel::Info, "hello"); @@ -194,7 +194,7 @@ mod tests { } #[test] - fn feed_cont_completion() { + fn a_continuation_completes_the_held_line() { let mut decoder = LogDecoder::new(); decoder.feed(IncomingLogLevel::Info, "hello "); let result = decoder.feed(IncomingLogLevel::Cont, "world\n"); @@ -212,7 +212,7 @@ mod tests { } #[test] - fn feed_multi_part_cont() { + fn continuations_join_until_the_newline() { let mut decoder = LogDecoder::new(); decoder.feed(IncomingLogLevel::Info, "part1 "); decoder.feed(IncomingLogLevel::Cont, "part2 "); @@ -231,7 +231,7 @@ mod tests { } #[test] - fn feed_non_cont_while_buffering() { + fn a_new_complete_line_flushes_the_held_line_as_stale() { let mut decoder = LogDecoder::new(); decoder.feed(IncomingLogLevel::Info, "stale"); let result = decoder.feed(IncomingLogLevel::Warn, "fresh\n"); @@ -255,7 +255,7 @@ mod tests { } #[test] - fn feed_buffer_replacement() { + fn a_new_partial_line_flushes_the_held_line_and_is_held_in_its_place() { let mut decoder = LogDecoder::new(); decoder.feed(IncomingLogLevel::Info, "first"); let result = decoder.feed(IncomingLogLevel::Warn, "second"); @@ -285,7 +285,7 @@ mod tests { } #[test] - fn feed_orphan_cont() { + fn a_continuation_without_any_earlier_line_is_an_orphan_anomaly() { let mut decoder = LogDecoder::new(); let result = decoder.feed(IncomingLogLevel::Cont, "ghost\n"); @@ -302,7 +302,7 @@ mod tests { } #[test] - fn feed_orphan_cont_previous_level() { + fn an_orphan_continuation_takes_the_previous_line_level() { let mut decoder = LogDecoder::new(); decoder.feed(IncomingLogLevel::Warn, "complete\n"); let result = decoder.feed(IncomingLogLevel::Cont, "ghost\n"); @@ -320,70 +320,31 @@ mod tests { } #[test] - fn feed_none_level() { - let mut decoder = LogDecoder::new(); - let result = decoder.feed(IncomingLogLevel::None, "no-level\n"); + fn a_complete_line_keeps_the_level_it_arrived_with() { + let incoming_levels = [ + IncomingLogLevel::None, + IncomingLogLevel::Unknown(9999), + IncomingLogLevel::Debug, + IncomingLogLevel::Error, + ]; assert_eq!( - result, - DecodeResult { - output: DecodeOutput::Line(LogLine { - level: LogLevel::None, - text: "no-level".to_owned(), - }), - anomaly: None, - } - ); - } - - #[test] - fn feed_unknown_level() { - let mut decoder = LogDecoder::new(); - let result = decoder.feed(IncomingLogLevel::Unknown(9999), "weird\n"); - - assert_eq!( - result, - DecodeResult { - output: DecodeOutput::Line(LogLine { - level: LogLevel::Unknown(9999), - text: "weird".to_owned(), - }), - anomaly: None, - } - ); - } - - #[test] - fn feed_debug_level() { - let mut decoder = LogDecoder::new(); - let result = decoder.feed(IncomingLogLevel::Debug, "trace\n"); - - assert_eq!( - result, - DecodeResult { - output: DecodeOutput::Line(LogLine { - level: LogLevel::Debug, - text: "trace".to_owned(), - }), - anomaly: None, - } - ); - } - - #[test] - fn feed_error_level() { - let mut decoder = LogDecoder::new(); - let result = decoder.feed(IncomingLogLevel::Error, "boom\n"); - - assert_eq!( - result, - DecodeResult { - output: DecodeOutput::Line(LogLine { - level: LogLevel::Error, - text: "boom".to_owned(), - }), - anomaly: None, - } + incoming_levels.map(|incoming_level| LogDecoder::new().feed(incoming_level, "line\n")), + [ + LogLevel::None, + LogLevel::Unknown(9999), + LogLevel::Debug, + LogLevel::Error + ] + .map(|level| { + DecodeResult { + output: DecodeOutput::Line(LogLine { + level, + text: "line".to_owned(), + }), + anomaly: None, + } + }) ); } diff --git a/llama-cpp-log-decoder/src/log_level.rs b/llama-cpp-log-decoder/src/log_level.rs index 0fa157be0..3dc7d8a89 100644 --- a/llama-cpp-log-decoder/src/log_level.rs +++ b/llama-cpp-log-decoder/src/log_level.rs @@ -13,7 +13,7 @@ mod tests { use super::LogLevel; #[test] - fn unknown_variant_equality() { + fn unknown_levels_are_equal_only_with_the_same_raw_value() { assert_eq!(LogLevel::Unknown(42), LogLevel::Unknown(42)); assert_ne!(LogLevel::Unknown(42), LogLevel::Unknown(43)); } From 40895a39a2ccda80dffbc7992ea641f56e62040c Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sat, 31 Oct 2026 19:35:55 +0100 Subject: [PATCH 31/82] Name the GGUF key lookup and typed getter tests after what they verify --- llama-cpp-bindings/src/gguf_context.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama-cpp-bindings/src/gguf_context.rs b/llama-cpp-bindings/src/gguf_context.rs index 787c81291..e482a7fd8 100644 --- a/llama-cpp-bindings/src/gguf_context.rs +++ b/llama-cpp-bindings/src/gguf_context.rs @@ -663,7 +663,7 @@ mod tests { } #[test] - fn find_key_returns_valid_index_for_known_key() { + fn find_key_finds_a_known_key_at_a_non_negative_index() { let context = GgufContext::from_file(fixture_path()).unwrap(); let index = context.find_key("general.architecture"); @@ -745,7 +745,7 @@ mod tests { } #[test] - fn typed_values_round_trip_through_synthetic_fixture() { + fn typed_getters_read_the_values_a_gguf_file_stores() { let fixture = SyntheticGgufBuilder::default() .value( b"synthetic.i32_value", From 6ee66087a8926ac4256892d48601d4a9be3227cd Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sat, 31 Oct 2026 19:48:58 +0100 Subject: [PATCH 32/82] Return parsed tool call fragments and evaluated multimodal usage as named values instead of tuples --- .../src/evaluated_multimodal_usage.rs | 8 +++ llama-cpp-bindings-tests/src/lib.rs | 1 + .../tests/multimodal_usage_accounting.rs | 27 ++++++--- llama-cpp-bindings/src/tool_call_format.rs | 1 + .../src/tool_call_format/bracketed_args.rs | 13 ++++- .../src/tool_call_format/paired_quote_args.rs | 58 ++++++++++++++----- .../src/tool_call_format/parsed_prefix.rs | 5 ++ llama-cpp-test-harness-macros/src/expand.rs | 2 +- 8 files changed, 90 insertions(+), 25 deletions(-) create mode 100644 llama-cpp-bindings-tests/src/evaluated_multimodal_usage.rs create mode 100644 llama-cpp-bindings/src/tool_call_format/parsed_prefix.rs diff --git a/llama-cpp-bindings-tests/src/evaluated_multimodal_usage.rs b/llama-cpp-bindings-tests/src/evaluated_multimodal_usage.rs new file mode 100644 index 000000000..aa17a3440 --- /dev/null +++ b/llama-cpp-bindings-tests/src/evaluated_multimodal_usage.rs @@ -0,0 +1,8 @@ +use llama_cpp_bindings_types::token_usage::TokenUsage; + +use crate::chunk_token_breakdown::ChunkTokenBreakdown; + +pub struct EvaluatedMultimodalUsage { + pub usage: TokenUsage, + pub chunk_breakdown: ChunkTokenBreakdown, +} diff --git a/llama-cpp-bindings-tests/src/lib.rs b/llama-cpp-bindings-tests/src/lib.rs index 2ddc3445c..a7980c7ee 100644 --- a/llama-cpp-bindings-tests/src/lib.rs +++ b/llama-cpp-bindings-tests/src/lib.rs @@ -6,6 +6,7 @@ pub mod classify_sample_loop; pub mod classify_sample_loop_outcome; pub mod closed_reasoning_chat_session; pub mod context_after_decoding_text; +pub mod evaluated_multimodal_usage; pub mod fixtures_dir; pub mod get_weather_tools_json; pub mod ingested_generation; diff --git a/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs b/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs index bfc1bfbb1..eea291302 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs @@ -12,15 +12,15 @@ use llama_cpp_bindings::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; +use llama_cpp_bindings_tests::evaluated_multimodal_usage::EvaluatedMultimodalUsage; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; use llama_cpp_bindings_tests::prompt_question::PROMPT_QUESTION; -use llama_cpp_bindings_types::token_usage::TokenUsage; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; fn build_multimodal_chunks_and_eval_into_usage( fixture: &LlamaFixture<'_>, -) -> Result<(TokenUsage, ChunkTokenBreakdown)> { +) -> Result { let model = fixture.model; let mtmd_ctx = fixture .mtmd_context @@ -60,7 +60,10 @@ fn build_multimodal_chunks_and_eval_into_usage( }, )?; - Ok((classifier.into_usage(), expected)) + Ok(EvaluatedMultimodalUsage { + usage: classifier.into_usage(), + chunk_breakdown: expected, + }) } #[llama_test( @@ -73,7 +76,10 @@ fn build_multimodal_chunks_and_eval_into_usage( mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), )] fn prompt_tokens_match_text_chunk_total(fixture: &LlamaFixture<'_>) -> Result<()> { - let (usage, expected) = build_multimodal_chunks_and_eval_into_usage(fixture)?; + let EvaluatedMultimodalUsage { + usage, + chunk_breakdown: expected, + } = build_multimodal_chunks_and_eval_into_usage(fixture)?; if usage.prompt_tokens != expected.text { bail!( @@ -96,7 +102,10 @@ fn prompt_tokens_match_text_chunk_total(fixture: &LlamaFixture<'_>) -> Result<() mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), )] fn input_image_tokens_match_image_chunk_total(fixture: &LlamaFixture<'_>) -> Result<()> { - let (usage, expected) = build_multimodal_chunks_and_eval_into_usage(fixture)?; + let EvaluatedMultimodalUsage { + usage, + chunk_breakdown: expected, + } = build_multimodal_chunks_and_eval_into_usage(fixture)?; if usage.input_image_tokens != expected.image { bail!( @@ -119,7 +128,10 @@ fn input_image_tokens_match_image_chunk_total(fixture: &LlamaFixture<'_>) -> Res mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), )] fn input_audio_tokens_are_zero_for_image_only_input(fixture: &LlamaFixture<'_>) -> Result<()> { - let (usage, expected) = build_multimodal_chunks_and_eval_into_usage(fixture)?; + let EvaluatedMultimodalUsage { + usage, + chunk_breakdown: expected, + } = build_multimodal_chunks_and_eval_into_usage(fixture)?; if expected.audio != 0 { bail!( @@ -149,7 +161,8 @@ fn input_audio_tokens_are_zero_for_image_only_input(fixture: &LlamaFixture<'_>) fn completion_tokens_are_zero_after_eval_before_generation( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let (usage, _expected) = build_multimodal_chunks_and_eval_into_usage(fixture)?; + let EvaluatedMultimodalUsage { usage, .. } = + build_multimodal_chunks_and_eval_into_usage(fixture)?; if usage.completion_tokens() != 0 { bail!( diff --git a/llama-cpp-bindings/src/tool_call_format.rs b/llama-cpp-bindings/src/tool_call_format.rs index 0a67cfbdf..2e231e0ae 100644 --- a/llama-cpp-bindings/src/tool_call_format.rs +++ b/llama-cpp-bindings/src/tool_call_format.rs @@ -4,6 +4,7 @@ pub mod json_object; pub mod key_value_xml_tags; pub mod paired_quote_args; pub mod parse_step; +pub mod parsed_prefix; pub mod scalar_value_to_json; pub mod separator_split; pub mod tool_call_format_outcome; diff --git a/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs b/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs index e982c55d5..c8d57dcee 100644 --- a/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs +++ b/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs @@ -7,12 +7,13 @@ use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; use crate::error::bracketed_args_failure::BracketedArgsFailure; use crate::tool_call_format::consume_optional_prefix::consume_optional_prefix; use crate::tool_call_format::parse_step::ParseStep; +use crate::tool_call_format::parsed_prefix::ParsedPrefix; use crate::tool_call_format::separator_split::SeparatorSplit; fn consume_one_json_value<'body>( input: &'body str, tool_name: &str, -) -> Result<(serde_json::Value, &'body str), BracketedArgsFailure> { +) -> Result, BracketedArgsFailure> { let mut stream = serde_json::Deserializer::from_str(input).into_iter::(); let value = stream .next() @@ -25,7 +26,10 @@ fn consume_one_json_value<'body>( })?; let consumed = stream.byte_offset(); - Ok((value, &input[consumed..])) + Ok(ParsedPrefix { + value, + remaining: &input[consumed..], + }) } fn parse_one_call<'body>( @@ -52,7 +56,10 @@ fn parse_one_call<'body>( return Ok(ParseStep::Done); } - let (arguments_value, after_arguments) = consume_one_json_value(after_separator, &name)?; + let ParsedPrefix { + value: arguments_value, + remaining: after_arguments, + } = consume_one_json_value(after_separator, &name)?; let after_close = consume_optional_prefix(after_arguments, markers.close.as_str()); diff --git a/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs b/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs index ce7811d50..4156b735e 100644 --- a/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs +++ b/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs @@ -8,6 +8,7 @@ use llama_cpp_bindings_types::tool_call_value_quote::ToolCallValueQuote; use crate::error::paired_quote_failure::PairedQuoteFailure; use crate::tool_call_format::consume_optional_prefix::consume_optional_prefix; use crate::tool_call_format::parse_step::ParseStep; +use crate::tool_call_format::parsed_prefix::ParsedPrefix; use crate::tool_call_format::scalar_value_to_json::scalar_value_to_json; use crate::tool_call_format::separator_split::SeparatorSplit; @@ -35,7 +36,7 @@ fn find_bare_value_end(input: &str, close_marker: &str) -> usize { fn parse_one_key<'body>( input: &'body str, tool_name: &str, -) -> Result<(String, &'body str), PairedQuoteFailure> { +) -> Result, PairedQuoteFailure> { let Some((key_raw, after_colon)) = input.split_once(':') else { return Err(PairedQuoteFailure::UnclosedArgumentBlock { tool_name: tool_name.to_owned(), @@ -49,7 +50,10 @@ fn parse_one_key<'body>( }); } - Ok((key, after_colon)) + Ok(ParsedPrefix { + value: key, + remaining: after_colon, + }) } fn parse_one_value<'body>( @@ -58,7 +62,7 @@ fn parse_one_value<'body>( close_marker: &str, tool_name: &str, key: &str, -) -> Result<(serde_json::Value, &'body str), PairedQuoteFailure> { +) -> Result, PairedQuoteFailure> { let trimmed = input.trim_start(); if !value_quote.open.is_empty() @@ -74,14 +78,20 @@ fn parse_one_value<'body>( let value_text = after_open[..close_position].to_owned(); let after_close = &after_open[close_position + value_quote.close.len()..]; - return Ok((serde_json::Value::String(value_text), after_close)); + return Ok(ParsedPrefix { + value: serde_json::Value::String(value_text), + remaining: after_close, + }); } let bare_end = find_bare_value_end(trimmed, close_marker); let bare_text = trimmed[..bare_end].trim(); let value = bare_value_to_json(bare_text); - Ok((value, &trimmed[bare_end..])) + Ok(ParsedPrefix { + value, + remaining: &trimmed[bare_end..], + }) } fn parse_args_body<'body>( @@ -89,30 +99,44 @@ fn parse_args_body<'body>( value_quote: &ToolCallValueQuote, close_marker: &str, tool_name: &str, -) -> Result<(serde_json::Map, &'body str), PairedQuoteFailure> { +) -> Result>, PairedQuoteFailure> { let mut map = serde_json::Map::new(); let mut remaining = input.trim_start(); loop { if remaining.is_empty() { - return Ok((map, remaining)); + return Ok(ParsedPrefix { + value: map, + remaining, + }); } if !close_marker.is_empty() && let Some(after_close) = remaining.strip_prefix(close_marker) { - return Ok((map, after_close)); + return Ok(ParsedPrefix { + value: map, + remaining: after_close, + }); } - let (key, after_key) = parse_one_key(remaining, tool_name)?; - let (value, after_value) = - parse_one_value(after_key, value_quote, close_marker, tool_name, &key)?; + let ParsedPrefix { + value: key, + remaining: after_key, + } = parse_one_key(remaining, tool_name)?; + let ParsedPrefix { + value, + remaining: after_value, + } = parse_one_value(after_key, value_quote, close_marker, tool_name, &key)?; map.insert(key.clone(), value); remaining = after_value.trim_start(); if !close_marker.is_empty() && let Some(after_close) = remaining.strip_prefix(close_marker) { - return Ok((map, after_close)); + return Ok(ParsedPrefix { + value: map, + remaining: after_close, + }); } if let Some(after_comma) = remaining.strip_prefix(',') { remaining = after_comma.trim_start(); @@ -120,7 +144,10 @@ fn parse_args_body<'body>( } let Some(character) = remaining.chars().next() else { - return Ok((map, remaining)); + return Ok(ParsedPrefix { + value: map, + remaining, + }); }; return Err(PairedQuoteFailure::UnexpectedCharAfterValue { @@ -155,7 +182,10 @@ fn parse_one_call<'body>( return Ok(ParseStep::Done); } - let (args_object, after_args) = parse_args_body( + let ParsedPrefix { + value: args_object, + remaining: after_args, + } = parse_args_body( after_separator, &shape.value_quote, markers.close.as_str(), diff --git a/llama-cpp-bindings/src/tool_call_format/parsed_prefix.rs b/llama-cpp-bindings/src/tool_call_format/parsed_prefix.rs new file mode 100644 index 000000000..e29c60581 --- /dev/null +++ b/llama-cpp-bindings/src/tool_call_format/parsed_prefix.rs @@ -0,0 +1,5 @@ +#[derive(Debug, Eq, PartialEq)] +pub struct ParsedPrefix<'body, TValue> { + pub value: TValue, + pub remaining: &'body str, +} diff --git a/llama-cpp-test-harness-macros/src/expand.rs b/llama-cpp-test-harness-macros/src/expand.rs index cdcbc9e42..66ae505e3 100644 --- a/llama-cpp-test-harness-macros/src/expand.rs +++ b/llama-cpp-test-harness-macros/src/expand.rs @@ -370,7 +370,7 @@ mod tests { expansion_error( well_formed_attribute(), quote! { - fn my_test(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } + fn my_test(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } }, ), "llama_test functions must not be generic" From 7ab3deae3e93a50f18c76648370e94f918e4e738 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sat, 31 Oct 2026 23:31:54 +0100 Subject: [PATCH 33/82] Give every private helper type its own module so each file holds one struct or enum --- llama-cpp-bindings/src/bare_json_tool_call.rs | 10 + .../src/bare_json_tool_call_field.rs | 5 + .../src/bare_json_tool_call_probe.rs | 32 ++- .../src/bare_json_tool_call_probe_state.rs | 11 + .../src/chat_session_attribution.rs | 15 ++ llama-cpp-bindings/src/content_probe.rs | 23 +++ llama-cpp-bindings/src/grammar_matcher.rs | 24 +-- .../src/grammar_matcher_step_outcome.rs | 4 + llama-cpp-bindings/src/lib.rs | 8 + .../src/sampled_token_classifier.rs | 37 +--- .../src/streaming_json_probe.rs | 189 +----------------- .../src/streaming_json_probe_state.rs | 157 +++++++++++++++ llama-cpp-bindings/src/token_attribution.rs | 6 + .../src/attribute_accumulator.rs | 17 ++ llama-cpp-test-harness-macros/src/lib.rs | 1 + .../src/parsed_args.rs | 16 +- 16 files changed, 297 insertions(+), 258 deletions(-) create mode 100644 llama-cpp-bindings/src/bare_json_tool_call.rs create mode 100644 llama-cpp-bindings/src/bare_json_tool_call_field.rs create mode 100644 llama-cpp-bindings/src/bare_json_tool_call_probe_state.rs create mode 100644 llama-cpp-bindings/src/chat_session_attribution.rs create mode 100644 llama-cpp-bindings/src/content_probe.rs create mode 100644 llama-cpp-bindings/src/grammar_matcher_step_outcome.rs create mode 100644 llama-cpp-bindings/src/streaming_json_probe_state.rs create mode 100644 llama-cpp-bindings/src/token_attribution.rs create mode 100644 llama-cpp-test-harness-macros/src/attribute_accumulator.rs diff --git a/llama-cpp-bindings/src/bare_json_tool_call.rs b/llama-cpp-bindings/src/bare_json_tool_call.rs new file mode 100644 index 000000000..8efb29c27 --- /dev/null +++ b/llama-cpp-bindings/src/bare_json_tool_call.rs @@ -0,0 +1,10 @@ +use serde::Deserialize; +use serde::de::IgnoredAny; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BareJsonToolCall { + pub name: String, + #[serde(rename = "arguments")] + _arguments: Option, +} diff --git a/llama-cpp-bindings/src/bare_json_tool_call_field.rs b/llama-cpp-bindings/src/bare_json_tool_call_field.rs new file mode 100644 index 000000000..765eb3fe6 --- /dev/null +++ b/llama-cpp-bindings/src/bare_json_tool_call_field.rs @@ -0,0 +1,5 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum BareJsonToolCallField { + Arguments, + Name, +} diff --git a/llama-cpp-bindings/src/bare_json_tool_call_probe.rs b/llama-cpp-bindings/src/bare_json_tool_call_probe.rs index 8bb871b57..dd1f49552 100644 --- a/llama-cpp-bindings/src/bare_json_tool_call_probe.rs +++ b/llama-cpp-bindings/src/bare_json_tool_call_probe.rs @@ -1,20 +1,12 @@ use std::mem; +use crate::bare_json_tool_call_probe_state::BareJsonToolCallProbeState; use crate::ingest_outcome::IngestOutcome; use crate::json_probe_outcome::JsonProbeOutcome; use crate::sampled_token::SampledToken; use crate::streaming_json_probe::StreamingJsonProbe; use crate::token_piece::TokenPiece; -#[derive(Clone, Debug, Eq, PartialEq)] -enum ProbeState { - Idle, - Holding { - probe: StreamingJsonProbe, - held: Vec, - }, -} - const fn is_visible_content(outcome: &IngestOutcome) -> bool { matches!(outcome.sampled_token, SampledToken::Content(_)) && matches!(outcome.piece, TokenPiece::Visible(_)) @@ -26,13 +18,13 @@ const fn is_visible_content(outcome: &IngestOutcome) -> bool { /// else is released as the content it was. #[derive(Clone, Debug, Eq, PartialEq)] pub struct BareJsonToolCallProbe { - state: ProbeState, + state: BareJsonToolCallProbeState, } impl Default for BareJsonToolCallProbe { fn default() -> Self { Self { - state: ProbeState::Idle, + state: BareJsonToolCallProbeState::Idle, } } } @@ -41,12 +33,12 @@ impl BareJsonToolCallProbe { pub fn pass(&mut self, outcome: IngestOutcome, released: &mut Vec) { let is_content = is_visible_content(&outcome); - match mem::replace(&mut self.state, ProbeState::Idle) { - ProbeState::Holding { held, .. } if !is_content => { + match mem::replace(&mut self.state, BareJsonToolCallProbeState::Idle) { + BareJsonToolCallProbeState::Holding { held, .. } if !is_content => { released.extend(held); released.push(outcome); } - ProbeState::Holding { + BareJsonToolCallProbeState::Holding { mut probe, mut held, } => { @@ -55,13 +47,15 @@ impl BareJsonToolCallProbe { held.push(outcome); self.settle(probe_outcome, probe, held, released); } - ProbeState::Idle if is_content && outcome.piece.raw().trim_start().starts_with('{') => { + BareJsonToolCallProbeState::Idle + if is_content && outcome.piece.raw().trim_start().starts_with('{') => + { let mut probe = StreamingJsonProbe::default(); let probe_outcome = probe.feed(outcome.piece.raw()); self.settle(probe_outcome, probe, vec![outcome], released); } - ProbeState::Idle => released.push(outcome), + BareJsonToolCallProbeState::Idle => released.push(outcome), } } @@ -74,7 +68,7 @@ impl BareJsonToolCallProbe { ) { match probe_outcome { JsonProbeOutcome::StillPossiblyValid => { - self.state = ProbeState::Holding { probe, held }; + self.state = BareJsonToolCallProbeState::Holding { probe, held }; } JsonProbeOutcome::CompletedValid => { released.extend(held.into_iter().map(|outcome| IngestOutcome { @@ -88,7 +82,9 @@ impl BareJsonToolCallProbe { /// Releases every held token as the content it was. pub fn release(&mut self, released: &mut Vec) { - if let ProbeState::Holding { held, .. } = mem::replace(&mut self.state, ProbeState::Idle) { + if let BareJsonToolCallProbeState::Holding { held, .. } = + mem::replace(&mut self.state, BareJsonToolCallProbeState::Idle) + { released.extend(held); } } diff --git a/llama-cpp-bindings/src/bare_json_tool_call_probe_state.rs b/llama-cpp-bindings/src/bare_json_tool_call_probe_state.rs new file mode 100644 index 000000000..72e54df86 --- /dev/null +++ b/llama-cpp-bindings/src/bare_json_tool_call_probe_state.rs @@ -0,0 +1,11 @@ +use crate::ingest_outcome::IngestOutcome; +use crate::streaming_json_probe::StreamingJsonProbe; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum BareJsonToolCallProbeState { + Idle, + Holding { + probe: StreamingJsonProbe, + held: Vec, + }, +} diff --git a/llama-cpp-bindings/src/chat_session_attribution.rs b/llama-cpp-bindings/src/chat_session_attribution.rs new file mode 100644 index 000000000..74f3ed63f --- /dev/null +++ b/llama-cpp-bindings/src/chat_session_attribution.rs @@ -0,0 +1,15 @@ +use std::collections::VecDeque; + +use crate::chat_output::ChatOutput; +use crate::chat_session::ChatSession; +use crate::content_probe::ContentProbe; +use crate::pending_chat_piece::PendingChatPiece; +use crate::token::LlamaToken; + +pub struct ChatSessionAttribution<'session, 'model> { + pub session: &'session ChatSession<'model>, + pub preserved_tokens: Vec, + pub output: ChatOutput, + pub pending_pieces: VecDeque, + pub content_probe: ContentProbe, +} diff --git a/llama-cpp-bindings/src/content_probe.rs b/llama-cpp-bindings/src/content_probe.rs new file mode 100644 index 000000000..8a7d6a017 --- /dev/null +++ b/llama-cpp-bindings/src/content_probe.rs @@ -0,0 +1,23 @@ +use crate::bare_json_tool_call_probe::BareJsonToolCallProbe; +use crate::ingest_outcome::IngestOutcome; + +pub enum ContentProbe { + Disabled, + BareJson(BareJsonToolCallProbe), +} + +impl ContentProbe { + pub fn pass(&mut self, outcome: IngestOutcome, released: &mut Vec) { + match self { + Self::Disabled => released.push(outcome), + Self::BareJson(probe) => probe.pass(outcome, released), + } + } + + pub fn release(&mut self, released: &mut Vec) { + match self { + Self::Disabled => {} + Self::BareJson(probe) => probe.release(released), + } + } +} diff --git a/llama-cpp-bindings/src/grammar_matcher.rs b/llama-cpp-bindings/src/grammar_matcher.rs index a4cbeb430..c15b0037b 100644 --- a/llama-cpp-bindings/src/grammar_matcher.rs +++ b/llama-cpp-bindings/src/grammar_matcher.rs @@ -7,13 +7,9 @@ use llguidance::TokenParser; use llguidance::api::StopReason; use crate::error::grammar_runtime_error::GrammarRuntimeError; +use crate::grammar_matcher_step_outcome::GrammarMatcherStepOutcome; use crate::mask_outcome::MaskOutcome; -enum StepOutcome { - Produced(TValue), - BenignStop, -} - fn panic_payload_message(payload: &(dyn Any + Send)) -> String { if let Some(message) = payload.downcast_ref::<&'static str>() { return (*message).to_owned(); @@ -87,8 +83,8 @@ impl GrammarMatcher { /// state (distinct from a benign grammar completion). pub fn compute_mask(&mut self) -> Result { match self.run("compute_mask", TokenParser::compute_mask)? { - StepOutcome::Produced(mask) => Ok(MaskOutcome::Constrained(mask)), - StepOutcome::BenignStop => Ok(MaskOutcome::GrammarComplete), + GrammarMatcherStepOutcome::Produced(mask) => Ok(MaskOutcome::Constrained(mask)), + GrammarMatcherStepOutcome::BenignStop => Ok(MaskOutcome::GrammarComplete), } } @@ -98,7 +94,9 @@ impl GrammarMatcher { /// benign stop and yields `Ok(())`. pub fn consume_token(&mut self, token: u32) -> Result<(), GrammarRuntimeError> { match self.run("consume_token", |parser| parser.consume_token(token))? { - StepOutcome::Produced(_) | StepOutcome::BenignStop => Ok(()), + GrammarMatcherStepOutcome::Produced(_) | GrammarMatcherStepOutcome::BenignStop => { + Ok(()) + } } } @@ -106,7 +104,9 @@ impl GrammarMatcher { /// Returns [`GrammarRuntimeError`] when the parser cannot be reset. pub fn reset(&mut self) -> Result<(), GrammarRuntimeError> { match self.run("reset", TokenParser::reset)? { - StepOutcome::Produced(()) | StepOutcome::BenignStop => Ok(()), + GrammarMatcherStepOutcome::Produced(()) | GrammarMatcherStepOutcome::BenignStop => { + Ok(()) + } } } @@ -114,9 +114,9 @@ impl GrammarMatcher { &mut self, operation: &'static str, step: impl FnOnce(&mut TokenParser) -> Result, - ) -> Result, GrammarRuntimeError> { + ) -> Result, GrammarRuntimeError> { match catch_step_panic(operation, || step(&mut self.parser))? { - Ok(value) => Ok(StepOutcome::Produced(value)), + Ok(value) => Ok(GrammarMatcherStepOutcome::Produced(value)), Err(step_error) => { let detail = self .parser @@ -124,7 +124,7 @@ impl GrammarMatcher { .unwrap_or_else(|| step_error.to_string()); stop_reason_to_result(self.parser.stop_reason(), detail)?; - Ok(StepOutcome::BenignStop) + Ok(GrammarMatcherStepOutcome::BenignStop) } } } diff --git a/llama-cpp-bindings/src/grammar_matcher_step_outcome.rs b/llama-cpp-bindings/src/grammar_matcher_step_outcome.rs new file mode 100644 index 000000000..39e161a95 --- /dev/null +++ b/llama-cpp-bindings/src/grammar_matcher_step_outcome.rs @@ -0,0 +1,4 @@ +pub enum GrammarMatcherStepOutcome { + Produced(TValue), + BenignStop, +} diff --git a/llama-cpp-bindings/src/lib.rs b/llama-cpp-bindings/src/lib.rs index f7814c848..6cae4f81a 100644 --- a/llama-cpp-bindings/src/lib.rs +++ b/llama-cpp-bindings/src/lib.rs @@ -4,7 +4,10 @@ )] mod backend_device_index; +mod bare_json_tool_call; +mod bare_json_tool_call_field; pub mod bare_json_tool_call_probe; +mod bare_json_tool_call_probe_state; pub mod bare_json_tool_calls; pub mod batch_add_error; pub mod chat_assistant_prefill; @@ -17,6 +20,7 @@ pub mod chat_output_completeness; pub mod chat_piece_role; pub mod chat_prompt_part; pub mod chat_session; +mod chat_session_attribution; pub mod chat_session_options; pub mod chat_span; pub mod chat_span_kind; @@ -26,6 +30,7 @@ pub mod chat_template_tool_calls; pub mod chat_templates; pub mod chat_tool_choice; pub mod classified_sample; +mod content_probe; pub mod context; pub mod cpp_text_view; pub mod dry_sequence_breakers; @@ -37,6 +42,7 @@ pub mod gguf_context_error; pub mod gguf_tensor_f32; pub mod gguf_type; pub mod grammar_matcher; +mod grammar_matcher_step_outcome; pub mod ingest_outcome; pub mod invalid_numa_strategy; pub mod json_probe_outcome; @@ -74,6 +80,8 @@ pub mod sampling; pub mod sanitized_grammar; pub mod send_logs_to_log; pub mod streaming_json_probe; +mod streaming_json_probe_state; pub mod token; +mod token_attribution; pub mod token_piece; pub mod tool_call_format; diff --git a/llama-cpp-bindings/src/sampled_token_classifier.rs b/llama-cpp-bindings/src/sampled_token_classifier.rs index 947157336..8c17a0606 100644 --- a/llama-cpp-bindings/src/sampled_token_classifier.rs +++ b/llama-cpp-bindings/src/sampled_token_classifier.rs @@ -12,7 +12,9 @@ use crate::chat_output::ChatOutput; use crate::chat_output_completeness::ChatOutputCompleteness; use crate::chat_piece_role::ChatPieceRole; use crate::chat_session::ChatSession; +use crate::chat_session_attribution::ChatSessionAttribution; use crate::classified_sample::ClassifiedSample; +use crate::content_probe::ContentProbe; use crate::context::LlamaContext; use crate::error::eval_multimodal_chunks_error::EvalMultimodalChunksError; use crate::error::ingest_error::IngestError; @@ -33,42 +35,9 @@ use crate::sampled_token::SampledToken; use crate::sampled_token_section::SampledTokenSection; use crate::sampling::LlamaSampler; use crate::token::LlamaToken; +use crate::token_attribution::TokenAttribution; use crate::token_piece::TokenPiece; -enum ContentProbe { - Disabled, - BareJson(BareJsonToolCallProbe), -} - -impl ContentProbe { - fn pass(&mut self, outcome: IngestOutcome, released: &mut Vec) { - match self { - Self::Disabled => released.push(outcome), - Self::BareJson(probe) => probe.pass(outcome, released), - } - } - - fn release(&mut self, released: &mut Vec) { - match self { - Self::Disabled => {} - Self::BareJson(probe) => probe.release(released), - } - } -} - -struct ChatSessionAttribution<'session, 'model> { - session: &'session ChatSession<'model>, - preserved_tokens: Vec, - output: ChatOutput, - pending_pieces: VecDeque, - content_probe: ContentProbe, -} - -enum TokenAttribution<'session, 'model> { - Undeterminable, - ChatSession(ChatSessionAttribution<'session, 'model>), -} - const fn outcome_of(token: LlamaToken, role: ChatPieceRole, piece: String) -> IngestOutcome { IngestOutcome { sampled_token: SampledToken::in_section(role.section(), token), diff --git a/llama-cpp-bindings/src/streaming_json_probe.rs b/llama-cpp-bindings/src/streaming_json_probe.rs index 9be943071..a57f56cb4 100644 --- a/llama-cpp-bindings/src/streaming_json_probe.rs +++ b/llama-cpp-bindings/src/streaming_json_probe.rs @@ -1,196 +1,25 @@ use std::mem; -use serde::Deserialize; -use serde::de::IgnoredAny; - +use crate::bare_json_tool_call::BareJsonToolCall; use crate::json_probe_outcome::JsonProbeOutcome; - -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct BareJsonToolCall { - name: String, - #[serde(rename = "arguments")] - _arguments: Option, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum ToolCallField { - Arguments, - Name, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -enum ProbeState { - AwaitingObjectOpen, - AwaitingFirstKeyOrClose, - AwaitingKey, - InKey { - quoted_key: String, - escaped: bool, - }, - AwaitingColon(ToolCallField), - AwaitingValue(ToolCallField), - InName { - escaped: bool, - }, - InArguments { - depth: usize, - in_string: bool, - escaped: bool, - }, - AwaitingCommaOrClose, - Closed, - Failed, -} - -const fn is_json_whitespace(character: char) -> bool { - matches!(character, ' ' | '\t' | '\n' | '\r') -} - -fn field_named(quoted_key: &str) -> Option { - match serde_json::from_str::(quoted_key).ok()?.as_str() { - "arguments" => Some(ToolCallField::Arguments), - "name" => Some(ToolCallField::Name), - _ => None, - } -} +use crate::streaming_json_probe_state::StreamingJsonProbeState; fn is_named_tool_call(held_text: &str) -> bool { serde_json::from_str::(held_text) .is_ok_and(|tool_call| !tool_call.name.is_empty()) } -impl ProbeState { - fn advance(self, character: char) -> Self { - match self { - Self::AwaitingObjectOpen => match character { - '{' => Self::AwaitingFirstKeyOrClose, - _ if is_json_whitespace(character) => Self::AwaitingObjectOpen, - _ => Self::Failed, - }, - Self::AwaitingFirstKeyOrClose => match character { - '"' => Self::InKey { - quoted_key: String::from('"'), - escaped: false, - }, - '}' => Self::Closed, - _ if is_json_whitespace(character) => Self::AwaitingFirstKeyOrClose, - _ => Self::Failed, - }, - Self::AwaitingKey => match character { - '"' => Self::InKey { - quoted_key: String::from('"'), - escaped: false, - }, - _ if is_json_whitespace(character) => Self::AwaitingKey, - _ => Self::Failed, - }, - Self::InKey { - mut quoted_key, - escaped, - } => { - quoted_key.push(character); - - if escaped || character != '"' { - Self::InKey { - quoted_key, - escaped: !escaped && character == '\\', - } - } else { - field_named("ed_key).map_or(Self::Failed, Self::AwaitingColon) - } - } - Self::AwaitingColon(field) => match character { - ':' => Self::AwaitingValue(field), - _ if is_json_whitespace(character) => Self::AwaitingColon(field), - _ => Self::Failed, - }, - Self::AwaitingValue(field) => match (field, character) { - (ToolCallField::Name, '"') => Self::InName { escaped: false }, - (ToolCallField::Arguments, '{') => Self::InArguments { - depth: 1, - in_string: false, - escaped: false, - }, - _ if is_json_whitespace(character) => Self::AwaitingValue(field), - _ => Self::Failed, - }, - Self::InName { escaped } => { - if !escaped && character == '"' { - Self::AwaitingCommaOrClose - } else { - Self::InName { - escaped: !escaped && character == '\\', - } - } - } - Self::InArguments { - depth, - in_string, - escaped, - } => Self::advance_arguments(depth, in_string, escaped, character), - Self::AwaitingCommaOrClose => match character { - ',' => Self::AwaitingKey, - '}' => Self::Closed, - _ if is_json_whitespace(character) => Self::AwaitingCommaOrClose, - _ => Self::Failed, - }, - Self::Closed if is_json_whitespace(character) => Self::Closed, - Self::Closed | Self::Failed => Self::Failed, - } - } - - const fn advance_arguments( - depth: usize, - in_string: bool, - escaped: bool, - character: char, - ) -> Self { - if in_string { - return Self::InArguments { - depth, - in_string: escaped || character != '"', - escaped: !escaped && character == '\\', - }; - } - - match character { - '"' => Self::InArguments { - depth, - in_string: true, - escaped: false, - }, - '{' | '[' => Self::InArguments { - depth: depth + 1, - in_string: false, - escaped: false, - }, - '}' | ']' if depth == 1 => Self::AwaitingCommaOrClose, - '}' | ']' => Self::InArguments { - depth: depth - 1, - in_string: false, - escaped: false, - }, - _ => Self::InArguments { - depth, - in_string: false, - escaped: false, - }, - } - } -} - #[derive(Clone, Debug, Eq, PartialEq)] pub struct StreamingJsonProbe { held_text: String, - state: ProbeState, + state: StreamingJsonProbeState, } impl Default for StreamingJsonProbe { fn default() -> Self { Self { held_text: String::new(), - state: ProbeState::AwaitingObjectOpen, + state: StreamingJsonProbeState::AwaitingObjectOpen, } } } @@ -200,20 +29,22 @@ impl StreamingJsonProbe { self.held_text.push_str(piece); for character in piece.chars() { - let state = mem::replace(&mut self.state, ProbeState::Failed); + let state = mem::replace(&mut self.state, StreamingJsonProbeState::Failed); self.state = state.advance(character); - if self.state == ProbeState::Failed { + if self.state == StreamingJsonProbeState::Failed { return JsonProbeOutcome::Failed; } } match self.state { - ProbeState::Closed if is_named_tool_call(&self.held_text) => { + StreamingJsonProbeState::Closed if is_named_tool_call(&self.held_text) => { JsonProbeOutcome::CompletedValid } - ProbeState::Closed | ProbeState::Failed => JsonProbeOutcome::Failed, + StreamingJsonProbeState::Closed | StreamingJsonProbeState::Failed => { + JsonProbeOutcome::Failed + } _ => JsonProbeOutcome::StillPossiblyValid, } } diff --git a/llama-cpp-bindings/src/streaming_json_probe_state.rs b/llama-cpp-bindings/src/streaming_json_probe_state.rs new file mode 100644 index 000000000..72564d5ee --- /dev/null +++ b/llama-cpp-bindings/src/streaming_json_probe_state.rs @@ -0,0 +1,157 @@ +use crate::bare_json_tool_call_field::BareJsonToolCallField; + +const fn is_json_whitespace(character: char) -> bool { + matches!(character, ' ' | '\t' | '\n' | '\r') +} + +fn field_named(quoted_key: &str) -> Option { + match serde_json::from_str::(quoted_key).ok()?.as_str() { + "arguments" => Some(BareJsonToolCallField::Arguments), + "name" => Some(BareJsonToolCallField::Name), + _ => None, + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum StreamingJsonProbeState { + AwaitingObjectOpen, + AwaitingFirstKeyOrClose, + AwaitingKey, + InKey { + quoted_key: String, + escaped: bool, + }, + AwaitingColon(BareJsonToolCallField), + AwaitingValue(BareJsonToolCallField), + InName { + escaped: bool, + }, + InArguments { + depth: usize, + in_string: bool, + escaped: bool, + }, + AwaitingCommaOrClose, + Closed, + Failed, +} + +impl StreamingJsonProbeState { + pub fn advance(self, character: char) -> Self { + match self { + Self::AwaitingObjectOpen => match character { + '{' => Self::AwaitingFirstKeyOrClose, + _ if is_json_whitespace(character) => Self::AwaitingObjectOpen, + _ => Self::Failed, + }, + Self::AwaitingFirstKeyOrClose => match character { + '"' => Self::InKey { + quoted_key: String::from('"'), + escaped: false, + }, + '}' => Self::Closed, + _ if is_json_whitespace(character) => Self::AwaitingFirstKeyOrClose, + _ => Self::Failed, + }, + Self::AwaitingKey => match character { + '"' => Self::InKey { + quoted_key: String::from('"'), + escaped: false, + }, + _ if is_json_whitespace(character) => Self::AwaitingKey, + _ => Self::Failed, + }, + Self::InKey { + mut quoted_key, + escaped, + } => { + quoted_key.push(character); + + if escaped || character != '"' { + Self::InKey { + quoted_key, + escaped: !escaped && character == '\\', + } + } else { + field_named("ed_key).map_or(Self::Failed, Self::AwaitingColon) + } + } + Self::AwaitingColon(field) => match character { + ':' => Self::AwaitingValue(field), + _ if is_json_whitespace(character) => Self::AwaitingColon(field), + _ => Self::Failed, + }, + Self::AwaitingValue(field) => match (field, character) { + (BareJsonToolCallField::Name, '"') => Self::InName { escaped: false }, + (BareJsonToolCallField::Arguments, '{') => Self::InArguments { + depth: 1, + in_string: false, + escaped: false, + }, + _ if is_json_whitespace(character) => Self::AwaitingValue(field), + _ => Self::Failed, + }, + Self::InName { escaped } => { + if !escaped && character == '"' { + Self::AwaitingCommaOrClose + } else { + Self::InName { + escaped: !escaped && character == '\\', + } + } + } + Self::InArguments { + depth, + in_string, + escaped, + } => Self::advance_arguments(depth, in_string, escaped, character), + Self::AwaitingCommaOrClose => match character { + ',' => Self::AwaitingKey, + '}' => Self::Closed, + _ if is_json_whitespace(character) => Self::AwaitingCommaOrClose, + _ => Self::Failed, + }, + Self::Closed if is_json_whitespace(character) => Self::Closed, + Self::Closed | Self::Failed => Self::Failed, + } + } + + const fn advance_arguments( + depth: usize, + in_string: bool, + escaped: bool, + character: char, + ) -> Self { + if in_string { + return Self::InArguments { + depth, + in_string: escaped || character != '"', + escaped: !escaped && character == '\\', + }; + } + + match character { + '"' => Self::InArguments { + depth, + in_string: true, + escaped: false, + }, + '{' | '[' => Self::InArguments { + depth: depth + 1, + in_string: false, + escaped: false, + }, + '}' | ']' if depth == 1 => Self::AwaitingCommaOrClose, + '}' | ']' => Self::InArguments { + depth: depth - 1, + in_string: false, + escaped: false, + }, + _ => Self::InArguments { + depth, + in_string: false, + escaped: false, + }, + } + } +} diff --git a/llama-cpp-bindings/src/token_attribution.rs b/llama-cpp-bindings/src/token_attribution.rs new file mode 100644 index 000000000..c7bffd118 --- /dev/null +++ b/llama-cpp-bindings/src/token_attribution.rs @@ -0,0 +1,6 @@ +use crate::chat_session_attribution::ChatSessionAttribution; + +pub enum TokenAttribution<'session, 'model> { + Undeterminable, + ChatSession(ChatSessionAttribution<'session, 'model>), +} diff --git a/llama-cpp-test-harness-macros/src/attribute_accumulator.rs b/llama-cpp-test-harness-macros/src/attribute_accumulator.rs new file mode 100644 index 000000000..f1e1b7077 --- /dev/null +++ b/llama-cpp-test-harness-macros/src/attribute_accumulator.rs @@ -0,0 +1,17 @@ +use crate::parsed_load_mode::ParsedLoadMode; +use crate::parsed_source::ParsedSource; + +#[derive(Default)] +pub struct AttributeAccumulator { + pub model_source: Option, + pub mmproj_source: Option, + pub n_gpu_layers: Option, + pub load_mode: Option, + pub n_ctx: Option, + pub n_batch: Option, + pub n_ubatch: Option, + pub embeddings: Option, + pub n_seq_max: Option, + pub n_threads_batch: Option, + pub void_logs: Option, +} diff --git a/llama-cpp-test-harness-macros/src/lib.rs b/llama-cpp-test-harness-macros/src/lib.rs index 4b1d6b59a..83b02b9fe 100644 --- a/llama-cpp-test-harness-macros/src/lib.rs +++ b/llama-cpp-test-harness-macros/src/lib.rs @@ -3,6 +3,7 @@ deny(clippy::unwrap_used, clippy::expect_used, clippy::panic) )] +mod attribute_accumulator; mod expand; mod parsed_args; mod parsed_context_params; diff --git a/llama-cpp-test-harness-macros/src/parsed_args.rs b/llama-cpp-test-harness-macros/src/parsed_args.rs index 3d9b6c8bc..8cef8f5b4 100644 --- a/llama-cpp-test-harness-macros/src/parsed_args.rs +++ b/llama-cpp-test-harness-macros/src/parsed_args.rs @@ -11,6 +11,7 @@ use syn::parse::Parse; use syn::parse::ParseStream; use syn::punctuated::Punctuated; +use crate::attribute_accumulator::AttributeAccumulator; use crate::parsed_context_params::ParsedContextParams; use crate::parsed_load_mode::ParsedLoadMode; use crate::parsed_model_load_params::ParsedModelLoadParams; @@ -93,21 +94,6 @@ fn require(value: Option, field: &str, span: Span) -> syn::Resul value.ok_or_else(|| syn::Error::new(span, format!("missing required field `{field}`"))) } -#[derive(Default)] -struct AttributeAccumulator { - model_source: Option, - mmproj_source: Option, - n_gpu_layers: Option, - load_mode: Option, - n_ctx: Option, - n_batch: Option, - n_ubatch: Option, - embeddings: Option, - n_seq_max: Option, - n_threads_batch: Option, - void_logs: Option, -} - fn dispatch_field( accumulator: &mut AttributeAccumulator, identifier: &Ident, From 4a0c1578544e8cd4dcbaf4701611d1de4ad93b5e Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sun, 1 Nov 2026 03:30:15 +0100 Subject: [PATCH 34/82] Fix review findings: dynamic backend linking, empty token roles, live causal attention --- .../src/bindings_generation.rs | 50 ++++++++++++++++- .../src/link_directives.rs | 8 +-- .../tests/multimodal_chunk_evaluation.rs | 54 +++++++++++++++++++ .../tests/unconstrained_token_sampling.rs | 36 ++++++++++++- llama-cpp-bindings/src/chat_spans.rs | 18 ++++++- llama-cpp-bindings/src/context.rs | 21 ++++---- llama-cpp-bindings/src/context/params.rs | 25 --------- llama-cpp-bindings/src/mtmd/eval_chunk.rs | 3 +- 8 files changed, 171 insertions(+), 44 deletions(-) diff --git a/llama-cpp-bindings-build/src/bindings_generation.rs b/llama-cpp-bindings-build/src/bindings_generation.rs index 3e77ce1e3..b0f44fecd 100644 --- a/llama-cpp-bindings-build/src/bindings_generation.rs +++ b/llama-cpp-bindings-build/src/bindings_generation.rs @@ -65,7 +65,13 @@ const DEPRECATED_FUNCTIONS: &[&str] = &[ ]; const SUPERSEDED_ITEMS: &[&str] = &[ - "LLAMA_TOKEN_TYPE_.*", + "LLAMA_TOKEN_TYPE_BYTE", + "LLAMA_TOKEN_TYPE_CONTROL", + "LLAMA_TOKEN_TYPE_NORMAL", + "LLAMA_TOKEN_TYPE_UNDEFINED", + "LLAMA_TOKEN_TYPE_UNKNOWN", + "LLAMA_TOKEN_TYPE_UNUSED", + "LLAMA_TOKEN_TYPE_USER_DEFINED", "ggml_backend_offload_op", "ggml_backend_supports_buft", "ggml_backend_supports_op", @@ -77,7 +83,14 @@ const SUPERSEDED_ITEMS: &[&str] = &[ "llama_encode", "llama_get_embeddings", "llama_get_logits", - "llama_perf_.*", + "llama_perf_context", + "llama_perf_context_data", + "llama_perf_context_print", + "llama_perf_context_reset", + "llama_perf_sampler", + "llama_perf_sampler_data", + "llama_perf_sampler_print", + "llama_perf_sampler_reset", "llama_token_type", "mtmd_image_tokens_get_id", "mtmd_image_tokens_get_n_pos", @@ -192,7 +205,10 @@ impl BindingsGeneration { #[cfg(test)] mod tests { + use std::env; + use std::fs; use std::path::PathBuf; + use std::process; use enumflags2::BitFlags; @@ -303,6 +319,36 @@ mod tests { ); } + #[test] + fn generated_bindings_hide_superseded_enum_variants_and_keep_their_replacements() { + let output = env::temp_dir().join(format!( + "llama-cpp-bindings-build-bindings-{}.rs", + process::id() + )); + BindingsGeneration { + header: PathBuf::from(FIXTURE_BINDINGS).join("wrapper.h"), + clang_args: Vec::new(), + output: output.clone(), + } + .generate() + .unwrap(); + let bindings = fs::read_to_string(&output).unwrap(); + fs::remove_file(&output).unwrap(); + let declares = |constant: &str| { + bindings + .lines() + .any(|line| line.trim() == format!("pub const {constant}: ggml_prec = 0;")) + }; + + assert_eq!( + [ + declares("GGML_PREC_DEFAULT"), + declares("GGML_PREC_UNDEFINED") + ], + [false, true] + ); + } + #[test] fn bindings_that_cannot_be_written_are_reported() { assert_eq!( diff --git a/llama-cpp-bindings-build/src/link_directives.rs b/llama-cpp-bindings-build/src/link_directives.rs index ca5ca2ee8..6c77d70a0 100644 --- a/llama-cpp-bindings-build/src/link_directives.rs +++ b/llama-cpp-bindings-build/src/link_directives.rs @@ -127,9 +127,11 @@ fn native_libraries(context: &BuildContext, environment: &BuildEnvironment) -> V if environment.has(BuildFeature::Rocm) { libraries.push(built("ggml-hip")); } + + libraries.push(built("ggml-cpu")); } - libraries.extend(["ggml-cpu", "ggml-base", "ggml"].map(built)); + libraries.extend(["ggml-base", "ggml"].map(built)); libraries } @@ -390,7 +392,8 @@ mod tests { } #[test] - fn a_dynamic_build_links_shared_libraries_and_leaves_gpu_backends_to_runtime_loading() { + fn a_dynamic_backends_build_links_shared_libraries_and_leaves_every_backend_to_runtime_loading() + { assert_eq!( libraries(&linked(&cargo_environment( BuildFeature::DynamicLink @@ -403,7 +406,6 @@ mod tests { "llama-common", "mtmd", "llama", - "ggml-cpu", "ggml-base", "ggml", "stdc++" diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs index 83097e507..90421a6bb 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs @@ -3,9 +3,14 @@ use anyhow::Result; use anyhow::bail; use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::context::llama_attention_type::LlamaAttentionType; use llama_cpp_bindings::error::eval_multimodal_chunks_error::EvalMultimodalChunksError; use llama_cpp_bindings::error::sequence_id_out_of_range::SequenceIdOutOfRange; use llama_cpp_bindings::eval_multimodal_chunks_params::EvalMultimodalChunksParams; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::mtmd::fitted_mtmd_input_chunk::FittedMtmdInputChunk; use llama_cpp_bindings::mtmd::fitted_mtmd_input_chunks::FittedMtmdInputChunks; use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; @@ -16,6 +21,7 @@ use llama_cpp_bindings::mtmd::mtmd_input_chunk_type_error::MtmdInputChunkTypeErr use llama_cpp_bindings::mtmd::mtmd_input_chunks::MtmdInputChunks; use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; use llama_cpp_bindings::mtmd::non_causal_chunk_micro_batch_mismatch::NonCausalChunkMicroBatchMismatch; +use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; @@ -620,6 +626,54 @@ fn eval_chunks_supports_extreme_dimensions(fixture: &LlamaFixture<'_>) -> Result Ok(()) } +#[llama_test( + model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 2048, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "mmproj-F16.gguf"), +)] +fn gemma3_decoding_follows_the_causal_attention_the_mtmd_helper_leaves_on_the_context( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params) + .into_llama_context_params() + .with_attention_type(LlamaAttentionType::NonCausal), + )?; + let n_batch = i32::try_from(llama_ctx.n_batch())?; + let next_position = chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, false)?; + let tokens = fixture.model.str_to_token( + &"llama ".repeat(600), + AddBos::Never, + ParseSpecialTokens::Never, + )?; + let last_index = tokens.len() - 1; + let mut batch = LlamaBatch::new(ContextLimits::of(&llama_ctx)); + + for (index, token) in tokens.iter().enumerate() { + batch.add( + &SampledToken::Content(*token), + next_position + i32::try_from(index)?, + &[0], + index == last_index, + )?; + } + + assert!(batch.n_tokens().unsigned_abs() > llama_ctx.n_ubatch()); + assert_eq!(llama_ctx.decode(&batch), Ok(())); + + Ok(()) +} + #[llama_test( model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), n_gpu_layers = 999, diff --git a/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs b/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs index fcae981fb..e5e91bd72 100644 --- a/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs +++ b/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs @@ -130,6 +130,38 @@ fn a_greedy_sampler_selects_a_candidate_of_a_decoded_row(fixture: &LlamaFixture< Ok(()) } +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -151,9 +183,9 @@ fn sampling_the_last_decoded_row_yields_a_token(fixture: &LlamaFixture<'_>) -> R batch.add_sequence(&tokens, 0, false)?; context.decode(&batch)?; let mut sampler = LlamaSampler::chain([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; - let result = sampler.sample(&context, batch.n_tokens() - 1); + let token = sampler.sample(&context, batch.n_tokens() - 1)?; - assert!(result.is_ok()); + assert!((0..fixture.model.n_vocab()).contains(&token.0)); Ok(()) } diff --git a/llama-cpp-bindings/src/chat_spans.rs b/llama-cpp-bindings/src/chat_spans.rs index 36cacfa75..a44ab4e89 100644 --- a/llama-cpp-bindings/src/chat_spans.rs +++ b/llama-cpp-bindings/src/chat_spans.rs @@ -111,7 +111,9 @@ impl ChatSpans { self.segments .iter() .skip(self.first_segment_ending_after(bytes.start)) - .take_while(|segment| segment.bytes.start < bytes.end) + .take_while(|segment| { + segment.bytes.start.max(bytes.start) < segment.bytes.end.min(bytes.end) + }) .map(|segment| segment.role) .find(|role| matches!(role, ChatPieceRole::Text(_))) .or_else(|| { @@ -278,6 +280,20 @@ mod tests { ); } + #[test] + fn a_token_without_text_inside_a_text_span_is_a_marker_of_that_section() { + let spans = ChatSpans::new( + 10, + &[span(ChatSpanKind::Content, 0, 10)], + ChatOutputCompleteness::Finished, + ); + + assert_eq!( + spans.classify_token(&(4..4)), + Some(ChatPieceRole::Marker(SampledTokenSection::Content)) + ); + } + #[test] fn a_reasoning_block_byte_outside_reasoning_text_is_a_reasoning_marker() { let spans = ChatSpans::new( diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index e2bb287c5..343a516ed 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -415,7 +415,7 @@ pub struct LlamaContext<'model> { decoded_outputs: DecodedOutputs, batch_ext: NonNull, limits: ContextLimits, - decodes_batches_in_one_micro_batch: bool, + keeps_no_memory: bool, } impl Debug for LlamaContext<'_> { @@ -449,8 +449,6 @@ impl<'model> LlamaContext<'model> { llama_cpp_bindings_sys::llama_pooling_type(context.as_ptr()) }); let keeps_no_memory = memory_of(context).is_none(); - let attends_causally = - unsafe { llama_cpp_bindings_sys::llama_get_causal_attn(context.as_ptr()) }; Ok(Self { context, @@ -467,7 +465,7 @@ impl<'model> LlamaContext<'model> { n_seq_max: unsafe { llama_cpp_bindings_sys::llama_n_seq_max(context.as_ptr()) }, n_vocab: model.n_vocab(), }, - decodes_batches_in_one_micro_batch: keeps_no_memory || !attends_causally, + keeps_no_memory, }) } @@ -529,6 +527,11 @@ impl<'model> LlamaContext<'model> { self.limits.n_seq_max } + fn decodes_batches_in_one_micro_batch(&self) -> bool { + self.keeps_no_memory + || !unsafe { llama_cpp_bindings_sys::llama_get_causal_attn(self.context.as_ptr()) } + } + fn exceeds_micro_batch(&self, batch: &LlamaBatch) -> bool { i64::from(batch.n_tokens()) > i64::from(self.n_ubatch()) } @@ -576,10 +579,10 @@ impl<'model> LlamaContext<'model> { unsafe { llama_cpp_bindings_sys::llama_detach_threadpool(self.context.as_ptr()) } } - /// Records which rows the last decode wrote, for a decode llama.cpp ran on this context - /// without one of this context's batches, such as the mtmd helper decoding media chunks. - pub fn record_decoded_rows(&mut self, rows: DecodedOutputRows) { - self.decoded_outputs.rows = rows; + /// Records the rows a llama.cpp helper wrote when it decoded on this context without one of + /// this context's batches, such as the mtmd helper decoding media chunks. + pub fn record_helper_decode(&mut self, logits_last: bool) { + self.decoded_outputs.rows = DecodedOutputRows::of_helper_decode(logits_last); } fn record_batch_output_rows(&mut self, output_token_indices: &[i32]) { @@ -649,7 +652,7 @@ impl<'model> LlamaContext<'model> { /// /// - `DecodeError` if the decoding failed. pub fn decode(&mut self, batch: &LlamaBatch) -> Result<(), DecodeError> { - if self.decodes_batches_in_one_micro_batch && self.exceeds_micro_batch(batch) { + if self.decodes_batches_in_one_micro_batch() && self.exceeds_micro_batch(batch) { return Err(DecodeError::BatchExceedsMicroBatch { n_tokens: batch.n_tokens(), n_ubatch: self.n_ubatch(), diff --git a/llama-cpp-bindings/src/context/params.rs b/llama-cpp-bindings/src/context/params.rs index 0b6725bfd..4d56c554d 100644 --- a/llama-cpp-bindings/src/context/params.rs +++ b/llama-cpp-bindings/src/context/params.rs @@ -288,17 +288,6 @@ impl LlamaContextParams { self.context_params.yarn_orig_ctx } - #[must_use] - pub const fn with_no_perf(mut self, no_perf: bool) -> Self { - self.context_params.no_perf = no_perf; - self - } - - #[must_use] - pub const fn no_perf(&self) -> bool { - self.context_params.no_perf - } - #[must_use] pub const fn with_op_offload(mut self, op_offload: bool) -> Self { self.context_params.op_offload = op_offload; @@ -663,20 +652,6 @@ mod tests { assert_eq!(params.yarn_orig_ctx(), 4096); } - #[test] - fn with_no_perf_enables() { - let params = super::LlamaContextParams::default().with_no_perf(true); - - assert!(params.no_perf()); - } - - #[test] - fn with_no_perf_disables() { - let params = super::LlamaContextParams::default().with_no_perf(false); - - assert!(!params.no_perf()); - } - #[test] fn with_op_offload_enables() { let params = super::LlamaContextParams::default().with_op_offload(true); diff --git a/llama-cpp-bindings/src/mtmd/eval_chunk.rs b/llama-cpp-bindings/src/mtmd/eval_chunk.rs index d21d4ea39..ea3fa49e6 100644 --- a/llama-cpp-bindings/src/mtmd/eval_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/eval_chunk.rs @@ -9,7 +9,6 @@ use super::mtmd_context::MtmdContext; use super::mtmd_eval_error::MtmdEvalError; use super::mtmd_input_chunk::MtmdInputChunk; use crate::context::LlamaContext; -use crate::context::decoded_output_rows::DecodedOutputRows; fn eval_chunk_single_status_to_result( status: llama_cpp_bindings_sys::llama_rs_mtmd_eval_chunk_single_status, @@ -112,7 +111,7 @@ pub fn eval_chunk( out_llama_cpp_return_code, out_error, )?; - llama_ctx.record_decoded_rows(DecodedOutputRows::of_helper_decode(logits_last)); + llama_ctx.record_helper_decode(logits_last); Ok(next_position) } From fd6b71f05efc1a0997dcf0531a87c92c312e2108 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sun, 1 Nov 2026 03:53:20 +0100 Subject: [PATCH 35/82] Keep raw batch entries inside LlamaBatch and report undocumented llama.cpp codes --- llama-cpp-bindings/src/context.rs | 231 +-------------------- llama-cpp-bindings/src/llama_batch.rs | 282 +++++++++++++++++++++++++- 2 files changed, 273 insertions(+), 240 deletions(-) diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index 343a516ed..cf38ed0a5 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -242,85 +242,6 @@ fn batch_ext_init_status_to_result( } } -fn batch_ext_assign_entries_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_batch_ext_assign_entries_status, - out_error: *mut c_char, -) -> Result<(), BatchTransferError> { - let refused_validated_entry = |detail: &'static str| -> Result<(), BatchTransferError> { - Err(FfiContractError { - operation: "llama_rs_batch_ext_assign_entries", - detail, - } - .into()) - }; - - match status { - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_ERROR_STRING_ALLOCATION_FAILED => { - Err(BatchTransferError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_OUT_OF_MEMORY => { - Err(BatchTransferError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - out_error, - "llama_rs_batch_ext_assign_entries", - "reported a thrown C++ exception without an error message", - ) - }?; - Err(BatchTransferError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_BATCH_FULL => refused_validated_entry( - "refused an entry of a batch validated against its limits as full", - ), - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_TOKEN => { - refused_validated_entry("refused a token validated against the vocabulary") - } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_SEQUENCE_ID => { - refused_validated_entry("refused a sequence id validated against the context") - } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_RETURNED_UNKNOWN_CODE => { - refused_validated_entry("refused an entry with an undocumented code") - } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_POSITION_REJECTED => { - refused_validated_entry("refused the position of an entry it had just added") - } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OUTPUT_REJECTED => { - refused_validated_entry("refused the output flag of an entry it had just added") - } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_EMPTY_SEQ_IDS => { - refused_validated_entry("was given an entry without sequence ids") - } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_SEQ_IDS_OUT_OF_RANGE => { - refused_validated_entry( - "was given an entry whose sequence ids lie outside the sequence id buffer", - ) - } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_ENTRIES_ARG => { - refused_validated_entry("was given a null entries argument") - } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_BATCH_ARG => { - refused_validated_entry("was given a null batch argument") - } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_SEQ_IDS_ARG => { - refused_validated_entry("was given a null seq_ids argument") - } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG => { - refused_validated_entry("was given a null out_llama_cpp_return_code argument") - } - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_ERROR_ARG => { - refused_validated_entry("was given a null out_error argument") - } - other => Err(FfiStatusError { - operation: "llama_rs_batch_ext_assign_entries", - code: i64::from(other), - } - .into()), - } -} - fn process_status_to_result( status: llama_cpp_bindings_sys::llama_rs_process_status, out_llama_cpp_return_code: i32, @@ -603,21 +524,7 @@ impl<'model> LlamaContext<'model> { }); } - let mut out_llama_cpp_return_code: i32 = 0; - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_batch_ext_assign_entries( - self.batch_ext.as_ptr(), - batch.entries().as_ptr(), - batch.entries().len(), - batch.seq_ids().as_ptr(), - batch.seq_ids().len(), - &raw mut out_llama_cpp_return_code, - &raw mut out_error, - ) - }; - - batch_ext_assign_entries_status_to_result(status, out_error) + unsafe { batch.assign_to(self.batch_ext) } } fn process( @@ -1430,142 +1337,6 @@ mod batch_ext_init_status_to_result_tests { } } -#[cfg(test)] -mod batch_ext_assign_entries_status_to_result_tests { - use std::ptr; - - use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; - use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; - - use super::batch_ext_assign_entries_status_to_result; - use crate::error::batch_transfer_error::BatchTransferError; - - #[test] - fn adding_a_batch_entry_reports_each_refusal_of_a_validated_entry_as_a_contract_violation() { - for (status, detail) in [ - ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_BATCH_FULL, - "refused an entry of a batch validated against its limits as full", - ), - ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_TOKEN, - "refused a token validated against the vocabulary", - ), - ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_SEQUENCE_ID, - "refused a sequence id validated against the context", - ), - ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_RETURNED_UNKNOWN_CODE, - "refused an entry with an undocumented code", - ), - ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_POSITION_REJECTED, - "refused the position of an entry it had just added", - ), - ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OUTPUT_REJECTED, - "refused the output flag of an entry it had just added", - ), - ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_EMPTY_SEQ_IDS, - "was given an entry without sequence ids", - ), - ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_SEQ_IDS_OUT_OF_RANGE, - "was given an entry whose sequence ids lie outside the sequence id buffer", - ), - ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_ENTRIES_ARG, - "was given a null entries argument", - ), - ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_BATCH_ARG, - "was given a null batch argument", - ), - ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_SEQ_IDS_ARG, - "was given a null seq_ids argument", - ), - ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG, - "was given a null out_llama_cpp_return_code argument", - ), - ( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_ERROR_ARG, - "was given a null out_error argument", - ), - ] { - assert_eq!( - batch_ext_assign_entries_status_to_result(status, ptr::null_mut()), - Err(BatchTransferError::FfiContract(FfiContractError { - operation: "llama_rs_batch_ext_assign_entries", - detail, - })) - ); - } - } - - #[test] - fn adding_a_batch_entry_reports_memory_exhaustion() { - assert_eq!( - [ - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_ERROR_STRING_ALLOCATION_FAILED, - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_OUT_OF_MEMORY, - ] - .map(|status| batch_ext_assign_entries_status_to_result(status, ptr::null_mut())), - [ - Err(BatchTransferError::NotEnoughMemory), - Err(BatchTransferError::LlamaCppOutOfMemory), - ] - ); - } - - #[test] - fn adding_a_batch_entry_reports_an_exception_message() { - let out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"entry insertion threw".as_ptr()) - }; - - assert_eq!( - batch_ext_assign_entries_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_THREW_CXX_EXCEPTION, - out_error, - ), - Err(BatchTransferError::Reported { - message: "entry insertion threw".to_owned() - }) - ); - } - - #[test] - fn adding_a_batch_entry_reports_an_exception_without_a_message_as_a_contract_violation() { - assert_eq!( - batch_ext_assign_entries_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null_mut(), - ), - Err(BatchTransferError::FfiContract( - FfiContractError { - operation: "llama_rs_batch_ext_assign_entries", - detail: "reported a thrown C++ exception without an error message", - } - )) - ); - } - - #[test] - fn adding_a_batch_entry_preserves_an_unknown_status() { - assert_eq!( - batch_ext_assign_entries_status_to_result(255, ptr::null_mut()), - Err(BatchTransferError::FfiStatus(FfiStatusError { - operation: "llama_rs_batch_ext_assign_entries", - code: 255, - })) - ); - } -} - #[cfg(test)] mod process_status_to_result_tests { use std::ptr; diff --git a/llama-cpp-bindings/src/llama_batch.rs b/llama-cpp-bindings/src/llama_batch.rs index 09513e2e7..9677474ed 100644 --- a/llama-cpp-bindings/src/llama_batch.rs +++ b/llama-cpp-bindings/src/llama_batch.rs @@ -1,10 +1,102 @@ +use std::ffi::c_char; +use std::ptr; +use std::ptr::NonNull; + use llama_cpp_bindings_sys::{llama_pos, llama_rs_batch_entry, llama_seq_id}; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use crate::batch_add_error::BatchAddError; use crate::context::context_limits::ContextLimits; +use crate::error::batch_transfer_error::BatchTransferError; use crate::sampled_token::SampledToken; use crate::token::LlamaToken; +fn batch_ext_assign_entries_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_batch_ext_assign_entries_status, + out_llama_cpp_return_code: i32, + out_error: *mut c_char, +) -> Result<(), BatchTransferError> { + let refused_validated_entry = |detail: &'static str| -> Result<(), BatchTransferError> { + Err(FfiContractError { + operation: "llama_rs_batch_ext_assign_entries", + detail, + } + .into()) + }; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OK => Ok(()), + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_ERROR_STRING_ALLOCATION_FAILED => { + Err(BatchTransferError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_OUT_OF_MEMORY => { + Err(BatchTransferError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + "llama_rs_batch_ext_assign_entries", + "reported a thrown C++ exception without an error message", + ) + }?; + Err(BatchTransferError::Reported { message }) + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_BATCH_FULL => refused_validated_entry( + "refused an entry of a batch validated against its limits as full", + ), + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_TOKEN => { + refused_validated_entry("refused a token validated against the vocabulary") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_SEQUENCE_ID => { + refused_validated_entry("refused a sequence id validated against the context") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_RETURNED_UNKNOWN_CODE => { + Err(FfiStatusError { + operation: "llama_batch_ext_add_token", + code: i64::from(out_llama_cpp_return_code), + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_POSITION_REJECTED => { + refused_validated_entry("refused the position of an entry it had just added") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OUTPUT_REJECTED => { + refused_validated_entry("refused the output flag of an entry it had just added") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_EMPTY_SEQ_IDS => { + refused_validated_entry("was given an entry without sequence ids") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_SEQ_IDS_OUT_OF_RANGE => { + refused_validated_entry( + "was given an entry whose sequence ids lie outside the sequence id buffer", + ) + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_ENTRIES_ARG => { + refused_validated_entry("was given a null entries argument") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_BATCH_ARG => { + refused_validated_entry("was given a null batch argument") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_SEQ_IDS_ARG => { + refused_validated_entry("was given a null seq_ids argument") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG => { + refused_validated_entry("was given a null out_llama_cpp_return_code argument") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_ERROR_ARG => { + refused_validated_entry("was given a null out_error argument") + } + other => Err(FfiStatusError { + operation: "llama_rs_batch_ext_assign_entries", + code: i64::from(other), + } + .into()), + } +} + /// Tokens to decode together. Each entry names its sequences by a range of `seq_ids`, the layout /// llama.cpp receives the whole batch in, in one call. #[derive(Clone, Debug, Eq, PartialEq)] @@ -122,14 +214,35 @@ impl LlamaBatch { self.n_tokens } - #[must_use] - pub fn entries(&self) -> &[llama_rs_batch_entry] { - &self.entries - } + /// Hands every token of this batch to llama.cpp's `batch_ext` in one call, replacing the + /// tokens it held. + /// + /// # Safety + /// + /// `batch_ext` must point to a batch that llama.cpp initialized and has not freed. + /// + /// # Errors + /// + /// Returns [`BatchTransferError`] when llama.cpp refuses an entry or the wrapper fails. + pub unsafe fn assign_to( + &self, + batch_ext: NonNull, + ) -> Result<(), BatchTransferError> { + let mut out_llama_cpp_return_code: i32 = 0; + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_batch_ext_assign_entries( + batch_ext.as_ptr(), + self.entries.as_ptr(), + self.entries.len(), + self.seq_ids.as_ptr(), + self.seq_ids.len(), + &raw mut out_llama_cpp_return_code, + &raw mut out_error, + ) + }; - #[must_use] - pub fn seq_ids(&self) -> &[llama_seq_id] { - &self.seq_ids + batch_ext_assign_entries_status_to_result(status, out_llama_cpp_return_code, out_error) } #[must_use] @@ -185,7 +298,7 @@ mod tests { batch.add(&content(2), 1, &[0, 1], true).unwrap(); assert_eq!( - batch.entries(), + batch.entries, [ llama_rs_batch_entry { token: 1, @@ -203,7 +316,7 @@ mod tests { }, ] ); - assert_eq!(batch.seq_ids(), [0, 0, 1]); + assert_eq!(batch.seq_ids, [0, 0, 1]); assert_eq!(batch.output_token_indices(), [1]); assert_eq!(batch.n_tokens(), 2); } @@ -280,7 +393,7 @@ mod tests { assert_eq!(batch.output_token_indices(), [2]); assert_eq!( - batch.entries().last(), + batch.entries.last(), Some(&llama_rs_batch_entry { token: 30, pos: 2, @@ -313,3 +426,152 @@ mod tests { assert_eq!(batch.n_tokens(), 4); } } + +#[cfg(test)] +mod batch_ext_assign_entries_status_to_result_tests { + use std::ptr; + + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + + use super::batch_ext_assign_entries_status_to_result; + use crate::error::batch_transfer_error::BatchTransferError; + + #[test] + fn adding_a_batch_entry_reports_each_refusal_of_a_validated_entry_as_a_contract_violation() { + for (status, detail) in [ + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_BATCH_FULL, + "refused an entry of a batch validated against its limits as full", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_TOKEN, + "refused a token validated against the vocabulary", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_SEQUENCE_ID, + "refused a sequence id validated against the context", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_POSITION_REJECTED, + "refused the position of an entry it had just added", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OUTPUT_REJECTED, + "refused the output flag of an entry it had just added", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_EMPTY_SEQ_IDS, + "was given an entry without sequence ids", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_SEQ_IDS_OUT_OF_RANGE, + "was given an entry whose sequence ids lie outside the sequence id buffer", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_ENTRIES_ARG, + "was given a null entries argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_BATCH_ARG, + "was given a null batch argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_SEQ_IDS_ARG, + "was given a null seq_ids argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG, + "was given a null out_llama_cpp_return_code argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_ERROR_ARG, + "was given a null out_error argument", + ), + ] { + assert_eq!( + batch_ext_assign_entries_status_to_result(status, 0, ptr::null_mut()), + Err(BatchTransferError::FfiContract(FfiContractError { + operation: "llama_rs_batch_ext_assign_entries", + detail, + })) + ); + } + } + + #[test] + fn an_undocumented_code_llama_cpp_returns_for_an_entry_is_reported_with_its_value() { + assert_eq!( + batch_ext_assign_entries_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_RETURNED_UNKNOWN_CODE, + -7, + ptr::null_mut(), + ), + Err(BatchTransferError::FfiStatus(FfiStatusError { + operation: "llama_batch_ext_add_token", + code: -7, + })) + ); + } + + #[test] + fn adding_a_batch_entry_reports_memory_exhaustion() { + assert_eq!( + [ + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_ERROR_STRING_ALLOCATION_FAILED, + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_OUT_OF_MEMORY, + ] + .map(|status| batch_ext_assign_entries_status_to_result(status, 0, ptr::null_mut())), + [ + Err(BatchTransferError::NotEnoughMemory), + Err(BatchTransferError::LlamaCppOutOfMemory), + ] + ); + } + + #[test] + fn adding_a_batch_entry_reports_an_exception_message() { + let out_error = unsafe { + llama_cpp_bindings_sys::llama_rs_string_dup(c"entry insertion threw".as_ptr()) + }; + + assert_eq!( + batch_ext_assign_entries_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + out_error, + ), + Err(BatchTransferError::Reported { + message: "entry insertion threw".to_owned() + }) + ); + } + + #[test] + fn adding_a_batch_entry_reports_an_exception_without_a_message_as_a_contract_violation() { + assert_eq!( + batch_ext_assign_entries_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + ptr::null_mut(), + ), + Err(BatchTransferError::FfiContract( + FfiContractError { + operation: "llama_rs_batch_ext_assign_entries", + detail: "reported a thrown C++ exception without an error message", + } + )) + ); + } + + #[test] + fn adding_a_batch_entry_preserves_an_unknown_status() { + assert_eq!( + batch_ext_assign_entries_status_to_result(255, 0, ptr::null_mut()), + Err(BatchTransferError::FfiStatus(FfiStatusError { + operation: "llama_rs_batch_ext_assign_entries", + code: 255, + })) + ); + } +} From 597da6c99ad4775df23fa71420e149b59f2f381b Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sun, 1 Nov 2026 04:16:12 +0100 Subject: [PATCH 36/82] Name the metadata entry tests after what they check --- llama-cpp-bindings-tests/tests/model_metadata.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama-cpp-bindings-tests/tests/model_metadata.rs b/llama-cpp-bindings-tests/tests/model_metadata.rs index 759711ce7..bbddefdfb 100644 --- a/llama-cpp-bindings-tests/tests/model_metadata.rs +++ b/llama-cpp-bindings-tests/tests/model_metadata.rs @@ -71,7 +71,7 @@ fn meta_count_returns_positive(fixture: &LlamaFixture<'_>) { n_batch = 128, n_ubatch = 64, )] -fn meta_key_by_index_returns_valid_key(fixture: &LlamaFixture<'_>) -> Result<()> { +fn the_first_metadata_entry_has_a_nonempty_key(fixture: &LlamaFixture<'_>) -> Result<()> { let key = fixture.model.meta_key_by_index(0)?; assert_ne!(key, ""); Ok(()) @@ -109,7 +109,7 @@ fn meta_key_by_index_returns_valid_key(fixture: &LlamaFixture<'_>) -> Result<()> n_batch = 128, n_ubatch = 64, )] -fn meta_val_str_by_index_returns_valid_value(fixture: &LlamaFixture<'_>) -> Result<()> { +fn the_first_metadata_entry_has_a_nonempty_value(fixture: &LlamaFixture<'_>) -> Result<()> { let value = fixture.model.meta_val_str_by_index(0)?; assert_ne!(value, ""); Ok(()) From 3ac1d8b744313194277d66189648456a953fdbbb Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sun, 1 Nov 2026 05:26:41 +0100 Subject: [PATCH 37/82] Name every test after what it verifies, assert exact values, and accept negative i32 test attributes --- llama-cpp-bindings-build/src/android_ndk.rs | 34 +- .../src/bindings_generation.rs | 21 +- llama-cpp-bindings-build/src/build_context.rs | 4 +- .../src/build_environment.rs | 2 +- .../src/cmake_configuration.rs | 2 +- llama-cpp-bindings-build/src/cpu_tuning.rs | 14 +- .../src/link_directives.rs | 2 +- llama-cpp-bindings-build/src/target_os.rs | 42 +-- .../src/wrapper_compilation.rs | 16 +- llama-cpp-bindings-tests/src/chat_answer.rs | 18 +- llama-cpp-bindings-tests/src/fixtures_dir.rs | 14 +- llama-cpp-bindings-tests/src/lib.rs | 5 + .../src/logits_left_by.rs | 26 ++ llama-cpp-bindings-tests/src/normalize.rs | 10 + .../src/prompted_reasoning.rs | 25 ++ llama-cpp-bindings-tests/src/single_token.rs | 17 + .../src/two_sequence_context.rs | 19 + .../src/yes_or_no_prefixes.rs | 1 + .../tests/chat_answer_parsing.rs | 35 +- .../tests/chat_session_input_validation.rs | 2 +- .../tests/chat_template_lookup.rs | 9 +- .../tests/chat_template_tool_call_support.rs | 2 +- .../tests/completion_usage_accounting.rs | 13 +- .../tests/context_creation_limits.rs | 14 +- .../tests/decode_abort_control.rs | 8 +- .../tests/decode_batch_rejection.rs | 12 +- .../tests/detokenization.rs | 211 +++++------ llama-cpp-bindings-tests/tests/dry_sampler.rs | 28 +- .../tests/embedding_availability.rs | 17 + .../tests/embedding_generation.rs | 64 +--- .../tests/embedding_retrieval_errors.rs | 10 +- .../tests/encode_batch_rejection.rs | 6 +- .../tests/encoder_requirement.rs | 4 +- .../end_of_generation_token_detection.rs | 90 +---- .../tests/grammar_sampler.rs | 50 ++- .../tests/hidden_state_extraction.rs | 103 +++--- .../json_schema_constrained_generation.rs | 61 ++-- .../tests/lazy_grammar_sampler.rs | 68 ++-- .../tests/llguidance_sampler.rs | 205 +++++------ .../tests/logit_bias_sampler.rs | 15 +- .../tests/logits_availability.rs | 20 -- .../tests/lora_adapters.rs | 92 ++++- .../tests/memory_availability.rs | 54 ++- .../tests/memory_occupancy.rs | 33 +- ...emory_position_shift_rope_compatibility.rs | 27 +- .../tests/memory_priming.rs | 67 +++- .../tests/memory_sequence_edits.rs | 7 +- .../tests/model_loading_errors.rs | 109 +----- .../tests/model_metadata.rs | 53 ++- .../tests/multimodal_audio.rs | 137 ++++--- .../tests/multimodal_bitmap_construction.rs | 23 +- .../tests/multimodal_chunk_encoding.rs | 18 +- .../tests/multimodal_chunk_evaluation.rs | 110 +++--- .../tests/multimodal_chunk_tokenization.rs | 65 ++-- .../tests/multimodal_fusion.rs | 9 - .../multimodal_projector_capabilities.rs | 65 ++++ .../tests/multimodal_projector_loading.rs | 2 +- .../multimodal_reasoning_classification.rs | 90 +---- llama-cpp-bindings-tests/tests/reranking.rs | 122 ++----- .../tests/session_file_path_validation.rs | 89 +++-- .../tests/session_file_token_capacity.rs | 134 +------ .../tests/session_files.rs | 42 ++- .../tests/special_tokens.rs | 4 +- .../tests/state_snapshots.rs | 54 +-- .../tests/token_attributes.rs | 14 +- .../tests/token_candidates.rs | 29 -- .../tests/tokenization.rs | 68 +--- .../tests/unconstrained_token_sampling.rs | 55 ++- .../tests/vocabulary_enumeration.rs | 48 --- .../src/parsed_chat_message.rs | 30 +- llama-cpp-bindings-types/src/token_usage.rs | 90 ++--- .../src/chat_message_parse_outcome.rs | 18 +- llama-cpp-bindings/src/chat_session.rs | 3 +- llama-cpp-bindings/src/chat_spans.rs | 15 +- .../known_marker_candidates.rs | 112 ++++++ llama-cpp-bindings/src/context.rs | 46 +-- .../src/context/kv_cache_type.rs | 22 +- llama-cpp-bindings/src/context/memory.rs | 42 +-- llama-cpp-bindings/src/context/params.rs | 170 ++++----- llama-cpp-bindings/src/context/session.rs | 30 +- .../src/error/new_llama_chat_message_error.rs | 2 +- llama-cpp-bindings/src/gguf_context.rs | 51 ++- llama-cpp-bindings/src/grammar_matcher.rs | 16 +- .../src/json_schema_to_grammar.rs | 91 ++--- .../src/lazy_grammar_triggers.rs | 2 +- llama-cpp-bindings/src/llama_backend.rs | 26 +- .../src/llama_backend_device.rs | 2 +- .../src/llama_backend_device_type.rs | 2 +- llama-cpp-bindings/src/llama_batch.rs | 3 +- llama-cpp-bindings/src/llama_token_attrs.rs | 31 +- llama-cpp-bindings/src/load_backends.rs | 4 +- llama-cpp-bindings/src/load_backends_error.rs | 2 +- .../src/load_backends_from_path.rs | 18 +- llama-cpp-bindings/src/log_options.rs | 2 +- llama-cpp-bindings/src/model.rs | 56 +-- .../src/model/llama_chat_message.rs | 42 ++- .../src/model/llama_chat_template.rs | 11 +- .../src/model/llama_lora_adapter.rs | 12 +- llama-cpp-bindings/src/model/params.rs | 336 +++++++++--------- .../src/model/params/param_override_value.rs | 10 +- llama-cpp-bindings/src/model/split_mode.rs | 12 +- llama-cpp-bindings/src/mtmd/eval_chunk.rs | 13 +- llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs | 80 ++--- llama-cpp-bindings/src/mtmd/mtmd_context.rs | 113 ++++-- .../src/mtmd/mtmd_input_chunk.rs | 21 +- llama-cpp-bindings/src/sampled_token.rs | 2 +- llama-cpp-bindings/src/sampling.rs | 289 ++++++++------- llama-cpp-bindings/src/send_logs_to_log.rs | 35 +- .../src/streaming_json_probe.rs | 118 +----- llama-cpp-bindings/src/token/data_array.rs | 4 +- llama-cpp-bindings/src/tool_call_format.rs | 152 -------- .../src/tool_call_format/bracketed_args.rs | 28 +- .../src/tool_call_format/json_object.rs | 20 +- .../src/tool_call_format/paired_quote_args.rs | 22 -- .../src/tool_call_format/xml_function_tags.rs | 8 +- llama-cpp-error-recorder/src/error_scope.rs | 16 +- llama-cpp-gbnf/src/validate_gbnf.rs | 4 +- llama-cpp-log-decoder/src/log_decoder.rs | 18 - llama-cpp-test-harness-macros/src/lib.rs | 2 +- .../src/parsed_args.rs | 156 +++++--- .../src/parsed_source.rs | 12 +- llama-cpp-test-harness/src/context_params.rs | 46 +-- .../src/model_load_params.rs | 2 +- llama-cpp-test-harness/src/run.rs | 21 -- .../src/run_to_conclusions.rs | 26 +- .../tests/phase_conclusions.rs | 6 +- .../src/compile_commands_file.rs | 42 ++- .../src/wrapper_sources_response_file.rs | 17 +- .../tests/emit_compile_commands.rs | 2 +- 129 files changed, 2628 insertions(+), 2895 deletions(-) create mode 100644 llama-cpp-bindings-tests/src/logits_left_by.rs create mode 100644 llama-cpp-bindings-tests/src/prompted_reasoning.rs create mode 100644 llama-cpp-bindings-tests/src/single_token.rs create mode 100644 llama-cpp-bindings-tests/src/two_sequence_context.rs create mode 100644 llama-cpp-bindings-tests/src/yes_or_no_prefixes.rs diff --git a/llama-cpp-bindings-build/src/android_ndk.rs b/llama-cpp-bindings-build/src/android_ndk.rs index e62e6197e..64c2a6c1b 100644 --- a/llama-cpp-bindings-build/src/android_ndk.rs +++ b/llama-cpp-bindings-build/src/android_ndk.rs @@ -249,13 +249,6 @@ pub mod tests { ) .unwrap(); - assert_eq!( - [detected.android_platform(), detected.cmake_toolchain_file(),], - [ - "android-30".to_owned(), - format!("{FIXTURE_ANDROID_NDK}/build/cmake/android.toolchain.cmake"), - ] - ); assert_eq!( detected, AndroidNdk { @@ -270,7 +263,30 @@ pub mod tests { } #[test] - fn the_api_level_is_read_with_or_without_the_android_prefix_and_defaults_to_28() { + fn the_cmake_platform_and_toolchain_file_derive_from_the_api_level_and_ndk_path() { + let android_ndk = AndroidNdk { + ndk_path: "/ndk".to_owned(), + api_level: "30".to_owned(), + abi: "arm64-v8a", + sysroot: "/ndk/sysroot".to_owned(), + target_prefix: "aarch64-linux-android", + clang_builtin_includes: "/ndk/lib/clang/20/include".to_owned(), + }; + + assert_eq!( + [ + android_ndk.android_platform(), + android_ndk.cmake_toolchain_file() + ], + [ + "android-30".to_owned(), + "/ndk/build/cmake/android.toolchain.cmake".to_owned(), + ] + ); + } + + #[test] + fn an_unprefixed_api_level_is_read_as_is_and_a_missing_one_defaults_to_28() { let api_level_of = |environment: &BuildEnvironment| { AndroidNdk::detect( environment, @@ -298,7 +314,7 @@ pub mod tests { } #[test] - fn detection_reports_every_way_the_ndk_cannot_be_used() { + fn detection_names_the_ndk_input_that_is_missing_unreadable_or_unsupported() { let fixture_ndk = Path::new(FIXTURE_ANDROID_NDK); let missing_root = temporary_directory("missing-ndk-root"); diff --git a/llama-cpp-bindings-build/src/bindings_generation.rs b/llama-cpp-bindings-build/src/bindings_generation.rs index b0f44fecd..4608a9d2d 100644 --- a/llama-cpp-bindings-build/src/bindings_generation.rs +++ b/llama-cpp-bindings-build/src/bindings_generation.rs @@ -261,19 +261,26 @@ mod tests { } #[test] - fn fault_injection_and_android_targets_add_their_clang_arguments() { + fn fault_injection_adds_its_define_to_the_clang_arguments() { + assert_eq!( + planned(BitFlags::from(BuildFeature::FaultInjection), &[]).clang_args, + [ + llama_cpp_includes(), + vec![format!("-D{FAULT_INJECTION_DEFINE}")], + ] + .concat() + ); + } + + #[test] + fn android_targets_add_the_ndk_clang_arguments() { let toolchain = format!("{FIXTURE_ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64"); assert_eq!( - planned( - BitFlags::from(BuildFeature::FaultInjection), - &ANDROID_OVERRIDES - ) - .clang_args, + planned(BitFlags::empty(), &ANDROID_OVERRIDES).clang_args, [ llama_cpp_includes(), vec![ - format!("-D{FAULT_INJECTION_DEFINE}"), format!("--sysroot={toolchain}/sysroot"), "-D__ANDROID_API__=28".to_owned(), "-D__ANDROID__".to_owned(), diff --git a/llama-cpp-bindings-build/src/build_context.rs b/llama-cpp-bindings-build/src/build_context.rs index e251c4322..cd272fc01 100644 --- a/llama-cpp-bindings-build/src/build_context.rs +++ b/llama-cpp-bindings-build/src/build_context.rs @@ -165,7 +165,7 @@ pub mod tests { } #[test] - fn the_cargo_variables_and_features_describe_the_build() { + fn the_context_is_read_from_the_build_variables_and_the_dynamic_link_feature() { assert_eq!( context_of(&cargo_environment( BitFlags::from(BuildFeature::DynamicLink), @@ -189,7 +189,7 @@ pub mod tests { } #[test] - fn every_variable_cargo_sets_for_build_scripts_is_required() { + fn every_variable_cargo_sets_for_build_scripts_except_the_target_env_is_required() { let without = |name: &str| { let mut environment = cargo_environment(BitFlags::empty(), &[]); diff --git a/llama-cpp-bindings-build/src/build_environment.rs b/llama-cpp-bindings-build/src/build_environment.rs index daa86d862..998bfda58 100644 --- a/llama-cpp-bindings-build/src/build_environment.rs +++ b/llama-cpp-bindings-build/src/build_environment.rs @@ -136,7 +136,7 @@ pub mod tests { } #[test] - fn a_missing_variable_is_required_but_optional() { + fn a_missing_variable_fails_when_required_and_is_none_when_optional() { let environment = environment_of(BitFlags::empty(), []); assert_eq!(environment.optional("HOST").ok(), Some(None)); diff --git a/llama-cpp-bindings-build/src/cmake_configuration.rs b/llama-cpp-bindings-build/src/cmake_configuration.rs index 863d18dff..c27d99864 100644 --- a/llama-cpp-bindings-build/src/cmake_configuration.rs +++ b/llama-cpp-bindings-build/src/cmake_configuration.rs @@ -481,7 +481,7 @@ mod tests { } #[test] - fn variables_the_configuration_reads_must_be_present_and_unicode() { + fn the_configuration_refuses_non_unicode_variables_and_a_missing_ggml_dir() { let context = context_of(&cargo_environment(BitFlags::empty(), &[])); let refusal = |features: BitFlags, name: &str| { CmakeConfiguration::of( diff --git a/llama-cpp-bindings-build/src/cpu_tuning.rs b/llama-cpp-bindings-build/src/cpu_tuning.rs index 88ccbc336..4a88c6575 100644 --- a/llama-cpp-bindings-build/src/cpu_tuning.rs +++ b/llama-cpp-bindings-build/src/cpu_tuning.rs @@ -149,8 +149,11 @@ mod target_cpu_of_tests { #[test] fn an_attached_codegen_target_cpu_is_found() { assert_eq!( - target_cpu_of("-Ctarget-cpu=native"), - Some("native".to_owned()) + [ + target_cpu_of("-Ctarget-cpu=native"), + target_cpu_of("--codegen=target-cpu=znver4"), + ], + [Some("native".to_owned()), Some("znver4".to_owned())] ); } @@ -160,13 +163,8 @@ mod target_cpu_of_tests { [ target_cpu_of("-C\x1ftarget-cpu=znver4"), target_cpu_of("--codegen\x1ftarget-cpu=znver4"), - target_cpu_of("--codegen=target-cpu=znver4"), ], - [ - Some("znver4".to_owned()), - Some("znver4".to_owned()), - Some("znver4".to_owned()), - ] + [Some("znver4".to_owned()), Some("znver4".to_owned())] ); } diff --git a/llama-cpp-bindings-build/src/link_directives.rs b/llama-cpp-bindings-build/src/link_directives.rs index 6c77d70a0..2b97369fa 100644 --- a/llama-cpp-bindings-build/src/link_directives.rs +++ b/llama-cpp-bindings-build/src/link_directives.rs @@ -615,7 +615,7 @@ mod tests { } #[test] - fn library_directories_an_enabled_feature_needs_must_exist() { + fn library_directories_an_enabled_feature_needs_must_be_set_and_exist() { let missing_directory = env::temp_dir().join(format!( "llama-cpp-bindings-build-missing-{}", process::id() diff --git a/llama-cpp-bindings-build/src/target_os.rs b/llama-cpp-bindings-build/src/target_os.rs index 7a012d06c..e3b46c781 100644 --- a/llama-cpp-bindings-build/src/target_os.rs +++ b/llama-cpp-bindings-build/src/target_os.rs @@ -106,28 +106,30 @@ mod tests { } #[test] - fn an_unsupported_or_unusable_target_is_refused() { + fn an_unsupported_target_os_is_refused() { assert_eq!( - [ - detected("freebsd"), - TargetOs::detect( - &environment_of( - BitFlags::empty(), - [("CARGO_CFG_TARGET_OS", "android".into())] - ), - "aarch64-linux-android", - "aarch64", - HostOs::Linux - ) - .map_err(|error| error.to_string()), - ], - [ - Err("unsupported target operating system: freebsd".to_owned()), - Err( - "Android NDK not found for target aarch64-linux-android. Set ANDROID_NDK_HOME." - .to_owned() + detected("freebsd"), + Err("unsupported target operating system: freebsd".to_owned()) + ); + } + + #[test] + fn an_android_target_without_an_ndk_is_refused() { + assert_eq!( + TargetOs::detect( + &environment_of( + BitFlags::empty(), + [("CARGO_CFG_TARGET_OS", "android".into())] ), - ] + "aarch64-linux-android", + "aarch64", + HostOs::Linux + ) + .map_err(|error| error.to_string()), + Err( + "Android NDK not found for target aarch64-linux-android. Set ANDROID_NDK_HOME." + .to_owned() + ) ); } } diff --git a/llama-cpp-bindings-build/src/wrapper_compilation.rs b/llama-cpp-bindings-build/src/wrapper_compilation.rs index 0c332653d..c244f3f75 100644 --- a/llama-cpp-bindings-build/src/wrapper_compilation.rs +++ b/llama-cpp-bindings-build/src/wrapper_compilation.rs @@ -142,14 +142,24 @@ mod tests { } #[test] - fn fault_injection_defines_its_switch_and_a_static_android_stl_is_linked_separately() { + fn fault_injection_defines_its_switch() { + assert_eq!( + planned(BitFlags::from(BuildFeature::FaultInjection), &[]), + WrapperCompilation { + defines: vec![FAULT_INJECTION_DEFINE], + ..linux_compilation() + } + ); + } + + #[test] + fn a_static_android_stl_is_linked_separately() { assert_eq!( planned( - BuildFeature::FaultInjection | BuildFeature::StaticStdcxx, + BitFlags::from(BuildFeature::StaticStdcxx), &ANDROID_OVERRIDES ), WrapperCompilation { - defines: vec![FAULT_INJECTION_DEFINE], cpp_stdlib_linked_separately: true, target_triple: "aarch64-linux-android".to_owned(), ..linux_compilation() diff --git a/llama-cpp-bindings-tests/src/chat_answer.rs b/llama-cpp-bindings-tests/src/chat_answer.rs index 5b2f9ce3a..a9af2d715 100644 --- a/llama-cpp-bindings-tests/src/chat_answer.rs +++ b/llama-cpp-bindings-tests/src/chat_answer.rs @@ -3,9 +3,7 @@ use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings_types::token_usage::TokenUsage; use crate::classify_sample_loop_outcome::ClassifySampleLoopOutcome; -use crate::recognized_answer::recognized_answer; - -const REASONING_PROBE: &str = "x"; +use crate::prompted_reasoning::prompted_reasoning; fn thinking_tags(session: &ChatSession<'_>) -> Vec { let mut tags = session @@ -21,20 +19,6 @@ fn thinking_tags(session: &ChatSession<'_>) -> Vec { tags } -fn prompted_reasoning(session: &ChatSession<'_>) -> String { - let probed = recognized_answer(session, REASONING_PROBE, ChatOutputCompleteness::Partial); - - if probed.content == REASONING_PROBE && probed.reasoning_content.is_empty() { - return String::new(); - } - - probed - .reasoning_content - .strip_suffix(REASONING_PROBE) - .expect("the probe must continue either the prompt's reasoning or its content") - .to_owned() -} - /// A generated answer to a chat session, with what the classifier attributed to each section. #[derive(Debug)] pub struct ChatAnswer { diff --git a/llama-cpp-bindings-tests/src/fixtures_dir.rs b/llama-cpp-bindings-tests/src/fixtures_dir.rs index 4683561fd..d92932e91 100644 --- a/llama-cpp-bindings-tests/src/fixtures_dir.rs +++ b/llama-cpp-bindings-tests/src/fixtures_dir.rs @@ -7,18 +7,14 @@ pub fn fixtures_dir() -> PathBuf { #[cfg(test)] mod tests { - use std::path::PathBuf; - use super::fixtures_dir; #[test] - fn resolves_to_the_fixtures_directory_inside_the_manifest() { - let expected = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fixtures"); - - assert_eq!(fixtures_dir(), expected); - assert!( - fixtures_dir().is_dir(), - "the fixtures directory the multimodal tests read from must exist" + fn resolves_to_the_directory_holding_every_media_fixture() { + assert_eq!( + ["llamas.jpg", "orange_cat.wav", "quick_brown_fox.wav"] + .map(|file_name| fixtures_dir().join(file_name).is_file()), + [true, true, true] ); } } diff --git a/llama-cpp-bindings-tests/src/lib.rs b/llama-cpp-bindings-tests/src/lib.rs index a7980c7ee..db5eb248d 100644 --- a/llama-cpp-bindings-tests/src/lib.rs +++ b/llama-cpp-bindings-tests/src/lib.rs @@ -10,6 +10,7 @@ pub mod evaluated_multimodal_usage; pub mod fixtures_dir; pub mod get_weather_tools_json; pub mod ingested_generation; +pub mod logits_left_by; pub mod media_question_chat_session; pub mod normalize; pub mod paris_weather_tool_call; @@ -17,7 +18,11 @@ pub mod penalized_nucleus_sampler; pub mod prime_memory; pub mod prime_memory_with; pub mod prompt_question; +pub mod prompted_reasoning; pub mod recognized_answer; +pub mod single_token; pub mod tokenize_prompt_with_synthetic_image; +pub mod two_sequence_context; pub mod user_turn_chat_session; pub mod weather_chat_session; +pub mod yes_or_no_prefixes; diff --git a/llama-cpp-bindings-tests/src/logits_left_by.rs b/llama-cpp-bindings-tests/src/logits_left_by.rs new file mode 100644 index 000000000..d69b0b93a --- /dev/null +++ b/llama-cpp-bindings-tests/src/logits_left_by.rs @@ -0,0 +1,26 @@ +use anyhow::Result; + +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_bindings::token::data::LlamaTokenData; +use llama_cpp_bindings::token::data_array::LlamaTokenDataArray; + +/// # Errors +/// Forwards [`LlamaSampler::apply`] errors verbatim. +pub fn logits_left_by(sampler: &LlamaSampler, candidates: &[LlamaToken]) -> Result> { + let mut token_data_array = LlamaTokenDataArray::new( + candidates + .iter() + .map(|candidate| LlamaTokenData::new(*candidate, 0.0, 0.0)) + .collect(), + false, + ); + + sampler.apply(&mut token_data_array)?; + + Ok(token_data_array + .data + .iter() + .map(LlamaTokenData::logit) + .collect()) +} diff --git a/llama-cpp-bindings-tests/src/normalize.rs b/llama-cpp-bindings-tests/src/normalize.rs index f8fd7695c..a4ac0bbf7 100644 --- a/llama-cpp-bindings-tests/src/normalize.rs +++ b/llama-cpp-bindings-tests/src/normalize.rs @@ -7,3 +7,13 @@ pub fn normalize(input: &[f32]) -> Vec { input.iter().map(|&value| value / magnitude).collect() } + +#[cfg(test)] +mod tests { + use super::normalize; + + #[test] + fn scales_a_vector_to_unit_length() { + assert_eq!(normalize(&[3.0, 4.0]), vec![0.6, 0.8]); + } +} diff --git a/llama-cpp-bindings-tests/src/prompted_reasoning.rs b/llama-cpp-bindings-tests/src/prompted_reasoning.rs new file mode 100644 index 000000000..1fcac1d80 --- /dev/null +++ b/llama-cpp-bindings-tests/src/prompted_reasoning.rs @@ -0,0 +1,25 @@ +use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; +use llama_cpp_bindings::chat_session::ChatSession; + +use crate::recognized_answer::recognized_answer; + +const REASONING_PROBE: &str = "x"; + +/// # Panics +/// +/// Panics when the parser fails on or does not recognize the probe, or when the probe continues +/// neither the prompt's reasoning nor its content. +#[must_use] +pub fn prompted_reasoning(session: &ChatSession<'_>) -> String { + let probed = recognized_answer(session, REASONING_PROBE, ChatOutputCompleteness::Partial); + + if probed.content == REASONING_PROBE && probed.reasoning_content.is_empty() { + return String::new(); + } + + probed + .reasoning_content + .strip_suffix(REASONING_PROBE) + .expect("the probe must continue either the prompt's reasoning or its content") + .to_owned() +} diff --git a/llama-cpp-bindings-tests/src/single_token.rs b/llama-cpp-bindings-tests/src/single_token.rs new file mode 100644 index 000000000..0c9aac2fd --- /dev/null +++ b/llama-cpp-bindings-tests/src/single_token.rs @@ -0,0 +1,17 @@ +use anyhow::Result; +use anyhow::bail; + +use llama_cpp_bindings::model::LlamaModel; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::token::LlamaToken; + +/// # Errors +/// Forwards tokenization errors, and fails when `text` tokenizes into anything but one token. +pub fn single_token(model: &LlamaModel, text: &str) -> Result { + let [token] = model.str_to_token(text, AddBos::Never, ParseSpecialTokens::Always)?[..] else { + bail!("{text:?} must be a single token"); + }; + + Ok(token) +} diff --git a/llama-cpp-bindings-tests/src/two_sequence_context.rs b/llama-cpp-bindings-tests/src/two_sequence_context.rs new file mode 100644 index 000000000..1eef43cd1 --- /dev/null +++ b/llama-cpp-bindings-tests/src/two_sequence_context.rs @@ -0,0 +1,19 @@ +use anyhow::Result; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; + +/// # Errors +/// Forwards [`LlamaContext::from_model`] errors verbatim. +pub fn two_sequence_context<'model>( + fixture: &LlamaFixture<'model>, +) -> Result> { + Ok(LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params) + .into_llama_context_params() + .with_kv_unified(true) + .with_n_seq_max(2), + )?) +} diff --git a/llama-cpp-bindings-tests/src/yes_or_no_prefixes.rs b/llama-cpp-bindings-tests/src/yes_or_no_prefixes.rs new file mode 100644 index 000000000..9447d9de6 --- /dev/null +++ b/llama-cpp-bindings-tests/src/yes_or_no_prefixes.rs @@ -0,0 +1 @@ +pub const YES_OR_NO_PREFIXES: [&str; 5] = ["y", "ye", "yes", "n", "no"]; diff --git a/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs b/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs index bbdb82fe8..b5b440233 100644 --- a/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs +++ b/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs @@ -7,13 +7,19 @@ use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; use llama_cpp_bindings::error::parse_chat_message_error::ParseChatMessageError; use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; +use llama_cpp_bindings_tests::prompted_reasoning::prompted_reasoning; +use llama_cpp_bindings_tests::recognized_answer::recognized_answer; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; use llama_cpp_bindings_tests::weather_chat_session::weather_chat_session; +use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; use llama_cpp_fault_injection::armed_fault::ArmedFault; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; +const GENERATED_REASONING: &str = "step one, step two"; +const GENERATED_CONTENT: &str = "actual response"; + #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -94,20 +100,21 @@ fn parses_pure_content_response(fixture: &LlamaFixture<'_>) -> Result<()> { n_batch = 128, n_ubatch = 64, )] -fn parses_reasoning_section_into_reasoning_content(fixture: &LlamaFixture<'_>) -> Result<()> { - let input = "step one, step two\n\nactual response"; - let outcome = user_turn_chat_session(fixture.model, "hello", "[]", false)? - .parse(&ChatOutput::from(input), ChatOutputCompleteness::Finished)?; +fn parses_the_reasoning_before_the_end_tag_apart_from_the_content( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, "hello", "[]", true)?; + let parsed = recognized_answer( + &session, + &format!("{GENERATED_REASONING}{GENERATED_CONTENT}"), + ChatOutputCompleteness::Finished, + ); - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!("expected Recognized for reasoning section; got Unrecognized"); - }; - assert!( - parsed.reasoning_content.contains("step") || parsed.content.contains("step"), - "neither content nor reasoning contains 'step'; content={:?} reasoning={:?}", - parsed.content, - parsed.reasoning_content + assert_eq!( + parsed.reasoning_content, + format!("{}{GENERATED_REASONING}", prompted_reasoning(&session)) ); + assert_eq!(parsed.content, GENERATED_CONTENT); Ok(()) } @@ -144,14 +151,14 @@ fn parses_reasoning_section_into_reasoning_content(fixture: &LlamaFixture<'_>) - n_batch = 128, n_ubatch = 64, )] -fn parses_empty_input_yields_empty_message(fixture: &LlamaFixture<'_>) -> Result<()> { +fn parses_an_empty_answer_into_an_empty_message(fixture: &LlamaFixture<'_>) -> Result<()> { let outcome = user_turn_chat_session(fixture.model, "hello", "[]", false)? .parse(&ChatOutput::from(""), ChatOutputCompleteness::Finished)?; let ChatMessageParseOutcome::Recognized(parsed) = outcome else { bail!("expected Recognized for empty input; got Unrecognized"); }; - assert_eq!(parsed.tool_calls, Vec::::new()); + assert_eq!(parsed, ParsedChatMessage::default()); Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs index 91aa724a6..fc45fae32 100644 --- a/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs +++ b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs @@ -24,7 +24,7 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 64, n_ubatch = 64, )] -fn media_question_chat_session_refuses_a_question_with_a_nul_byte( +fn chat_message_reports_the_nul_byte_position_after_the_media_marker( fixture: &LlamaFixture<'_>, ) -> Result<()> { let Err(failure) = media_question_chat_session(fixture.model, "describe\0this") else { diff --git a/llama-cpp-bindings-tests/tests/chat_template_lookup.rs b/llama-cpp-bindings-tests/tests/chat_template_lookup.rs index 9db306a3e..ae5d1d49f 100644 --- a/llama-cpp-bindings-tests/tests/chat_template_lookup.rs +++ b/llama-cpp-bindings-tests/tests/chat_template_lookup.rs @@ -1,5 +1,7 @@ use std::ffi::CString; +use anyhow::Result; + use llama_cpp_bindings::error::chat_template_error::ChatTemplateError; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -36,9 +38,10 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 128, n_ubatch = 64, )] -fn chat_template_returns_non_empty(fixture: &LlamaFixture<'_>) { - let template = fixture.model.chat_template(None); - assert!(template.is_ok()); +fn default_chat_template_is_not_empty(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_ne!(fixture.model.chat_template(None)?.to_str()?, ""); + + Ok(()) } #[llama_test( diff --git a/llama-cpp-bindings-tests/tests/chat_template_tool_call_support.rs b/llama-cpp-bindings-tests/tests/chat_template_tool_call_support.rs index f0c5e8307..9b90a767e 100644 --- a/llama-cpp-bindings-tests/tests/chat_template_tool_call_support.rs +++ b/llama-cpp-bindings-tests/tests/chat_template_tool_call_support.rs @@ -11,7 +11,7 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 512, n_ubatch = 128 )] -fn embedding_model_chat_template_renders_tool_calls(fixture: &LlamaFixture<'_>) -> Result<()> { +fn embedding_model_chat_template_supports_tool_calls(fixture: &LlamaFixture<'_>) -> Result<()> { assert!(fixture.model.chat_templates()?.supports_tool_calls()); Ok(()) diff --git a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs index d2e9fd525..62eeea7cb 100644 --- a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs +++ b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs @@ -74,15 +74,10 @@ fn raw_prompt_completion_usage_matches_the_classified_tokens( let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens_list, 0, false)?; - assert_eq!(classifier.pending_prompt_tokens(), prompt_token_count); - assert_eq!(classifier.usage().prompt_tokens, 0); - ctx.decode(&batch) .with_context(|| "llama_decode() failed")?; - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - assert_eq!(classifier.usage().prompt_tokens, prompt_token_count); + classifier.commit_prompt_tokens(); let mut sampler = LlamaSampler::chain([LlamaSampler::dist(1234)?, LlamaSampler::greedy()?])?; let initial_position = batch.n_tokens(); @@ -207,13 +202,9 @@ fn chat_completion_usage_matches_the_classified_tokens(fixture: &LlamaFixture<'_ let mut batch = LlamaBatch::new(ContextLimits::of(&context)); classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; - assert_eq!(classifier.pending_prompt_tokens(), prompt_token_count); - assert_eq!(classifier.usage().prompt_tokens, 0); - context.decode(&batch)?; - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); + classifier.commit_prompt_tokens(); let mut sampler = LlamaSampler::greedy()?; let initial_position = batch.n_tokens(); diff --git a/llama-cpp-bindings-tests/tests/context_creation_limits.rs b/llama-cpp-bindings-tests/tests/context_creation_limits.rs index 72f723a8e..05d919230 100644 --- a/llama-cpp-bindings-tests/tests/context_creation_limits.rs +++ b/llama-cpp-bindings-tests/tests/context_creation_limits.rs @@ -39,10 +39,16 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 128, n_ubatch = 64, )] -fn new_context_with_huge_ctx_returns_null_error(fixture: &LlamaFixture<'_>) { - let result = fixture.build_context(); - - assert!(result.is_err()); +fn a_context_size_overflowing_llama_cpp_padding_is_unconstructible(fixture: &LlamaFixture<'_>) { + assert_eq!( + LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + ) + .err(), + Some(LlamaContextLoadError::Unconstructible) + ); } #[llama_test( diff --git a/llama-cpp-bindings-tests/tests/decode_abort_control.rs b/llama-cpp-bindings-tests/tests/decode_abort_control.rs index 71ba8f1ca..8a7da1161 100644 --- a/llama-cpp-bindings-tests/tests/decode_abort_control.rs +++ b/llama-cpp-bindings-tests/tests/decode_abort_control.rs @@ -65,9 +65,7 @@ fn set_abort_flag_false_allows_decode(fixture: &LlamaFixture<'_>) -> Result<()> let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - let result = context.decode(&batch); - - assert!(result.is_ok()); + assert_eq!(context.decode(&batch), Ok(())); Ok(()) } @@ -96,9 +94,7 @@ fn clear_abort_callback_allows_decode_with_flag_true(fixture: &LlamaFixture<'_>) let mut batch = LlamaBatch::new(ContextLimits::of(&context)); batch.add_sequence(&tokens, 0, false)?; - let result = context.decode(&batch); - - assert!(result.is_ok()); + assert_eq!(context.decode(&batch), Ok(())); Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs b/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs index 3f163878e..d0c579ed7 100644 --- a/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs +++ b/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs @@ -21,7 +21,7 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 2048, n_ubatch = 512, )] -fn decode_empty_batch_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { +fn decoding_an_empty_batch_is_refused_as_invalid(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = LlamaContext::from_model( fixture.model, fixture.backend, @@ -29,9 +29,7 @@ fn decode_empty_batch_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { )?; let batch = LlamaBatch::new(ContextLimits::of(&context)); - let result = context.decode(&batch); - - assert!(result.is_err()); + assert_eq!(context.decode(&batch), Err(DecodeError::BatchInvalid)); Ok(()) } @@ -44,7 +42,7 @@ fn decode_empty_batch_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { n_batch = 512, n_ubatch = 64, )] -fn decoding_more_tokens_than_the_micro_batch_with_non_causal_attention_returns_an_error( +fn decoding_more_tokens_than_the_micro_batch_with_non_causal_attention_is_refused( fixture: &LlamaFixture<'_>, ) -> Result<()> { let mut context = LlamaContext::from_model( @@ -83,7 +81,7 @@ fn decoding_more_tokens_than_the_micro_batch_with_non_causal_attention_returns_a n_ubatch = 64, embeddings = true, )] -fn decoding_more_tokens_than_the_micro_batch_on_an_encoder_returns_an_error( +fn decoding_more_tokens_than_the_micro_batch_on_an_encoder_is_refused( fixture: &LlamaFixture<'_>, ) -> Result<()> { let mut context = fixture.build_context()?; @@ -115,7 +113,7 @@ fn decoding_more_tokens_than_the_micro_batch_on_an_encoder_returns_an_error( n_batch = 2048, n_ubatch = 512, )] -fn decoding_reports_a_token_llama_cpp_cannot_add_to_its_batch( +fn decoding_reports_an_exception_thrown_while_transferring_the_batch( fixture: &LlamaFixture<'_>, ) -> Result<()> { let mut context = fixture.build_context()?; diff --git a/llama-cpp-bindings-tests/tests/detokenization.rs b/llama-cpp-bindings-tests/tests/detokenization.rs index c3c0f47b4..235140ffa 100644 --- a/llama-cpp-bindings-tests/tests/detokenization.rs +++ b/llama-cpp-bindings-tests/tests/detokenization.rs @@ -1,56 +1,20 @@ +use std::ffi::c_int; use std::num::NonZeroU16; use anyhow::Context as _; use anyhow::Result; +use anyhow::bail; use llama_cpp_bindings::error::token_to_string_error::TokenToStringError; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_bindings_tests::single_token::single_token; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_to_piece_bytes_returns_bytes_for_known_token(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let tokens = model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; - let bytes = model.token_to_piece_bytes(tokens[0], 32, false, None)?; - - assert_ne!(bytes, Vec::::new()); - Ok(()) -} +const PIECE_PROBE_BYTES: usize = 32; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), @@ -84,60 +48,15 @@ fn token_to_piece_bytes_returns_bytes_for_known_token(fixture: &LlamaFixture<'_> n_batch = 128, n_ubatch = 64, )] -fn token_to_piece_handles_large_token_requiring_buffer_resize(fixture: &LlamaFixture<'_>) { - let model = fixture.model; - let mut decoder = encoding_rs::UTF_8.new_decoder(); - - for (token, _) in model.tokens(true).take(200) { - let result = model.token_to_piece(&SampledToken::Content(token), &mut decoder, true, None); - assert!(result.is_ok()); - } -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_to_piece_bytes_insufficient_buffer_returns_error( +fn token_to_piece_bytes_returns_the_bytes_of_the_token_piece( fixture: &LlamaFixture<'_>, ) -> Result<()> { let model = fixture.model; let tokens = model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; - let result = model.token_to_piece_bytes(tokens[0], 1, false, None); - assert!( - result - .unwrap_err() - .to_string() - .contains("Insufficient Buffer Space") + assert_eq!( + model.token_to_piece_bytes(tokens[0], PIECE_PROBE_BYTES, false, None)?, + b"hello" ); Ok(()) } @@ -174,20 +93,41 @@ fn token_to_piece_bytes_insufficient_buffer_returns_error( n_batch = 128, n_ubatch = 64, )] -fn token_to_piece_accepts_a_count_of_leading_spaces_to_strip( +fn token_to_piece_returns_a_piece_longer_than_its_probe_buffer_whole( fixture: &LlamaFixture<'_>, ) -> Result<()> { let model = fixture.model; - let mut decoder = encoding_rs::UTF_8.new_decoder(); - let tokens = model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; - let result = model.token_to_piece( - &SampledToken::Content(tokens[0]), - &mut decoder, + let long_token = (0..model.n_vocab()) + .map(LlamaToken::new) + .find(|token| { + matches!( + model.token_to_piece_bytes(*token, PIECE_PROBE_BYTES, false, None), + Err(TokenToStringError::InsufficientBufferSpace(_)) + ) + }) + .context("the vocabulary has a piece longer than the probe buffer")?; + let Err(TokenToStringError::InsufficientBufferSpace(negated_piece_size)) = + model.token_to_piece_bytes(long_token, PIECE_PROBE_BYTES, false, None) + else { + bail!("the probe buffer must be too small for the long piece"); + }; + let long_piece = model.token_to_piece_bytes( + long_token, + usize::try_from(negated_piece_size.unsigned_abs())?, false, - NonZeroU16::new(1), - ); + None, + )?; + let mut decoder = encoding_rs::UTF_8.new_decoder(); - assert!(result.is_ok()); + assert_eq!( + model.token_to_piece( + &SampledToken::Content(long_token), + &mut decoder, + false, + None + )?, + String::from_utf8(long_piece)? + ); Ok(()) } @@ -223,19 +163,18 @@ fn token_to_piece_accepts_a_count_of_leading_spaces_to_strip( n_batch = 128, n_ubatch = 64, )] -fn token_to_piece_decodes_reasoning_variant(fixture: &LlamaFixture<'_>) -> Result<()> { +fn token_to_piece_bytes_with_a_too_small_buffer_reports_the_size_the_piece_needs( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let model = fixture.model; - let mut decoder = encoding_rs::UTF_8.new_decoder(); - let tokens = model.str_to_token("hi", AddBos::Never, ParseSpecialTokens::Always)?; - - let piece = model.token_to_piece( - &SampledToken::Reasoning(tokens[0]), - &mut decoder, - true, - None, - )?; + let tokens = model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; - assert_ne!(piece, ""); + assert_eq!( + model.token_to_piece_bytes(tokens[0], 1, false, None), + Err(TokenToStringError::InsufficientBufferSpace( + -c_int::try_from("hello".len())? + )) + ); Ok(()) } @@ -271,15 +210,22 @@ fn token_to_piece_decodes_reasoning_variant(fixture: &LlamaFixture<'_>) -> Resul n_batch = 128, n_ubatch = 64, )] -fn token_to_piece_decodes_tool_call_variant(fixture: &LlamaFixture<'_>) -> Result<()> { +fn token_to_piece_strips_the_requested_count_of_leading_spaces( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let model = fixture.model; let mut decoder = encoding_rs::UTF_8.new_decoder(); - let tokens = model.str_to_token("hi", AddBos::Never, ParseSpecialTokens::Always)?; + let tokens = model.str_to_token(" hello", AddBos::Never, ParseSpecialTokens::Always)?; - let piece = - model.token_to_piece(&SampledToken::ToolCall(tokens[0]), &mut decoder, true, None)?; - - assert_ne!(piece, ""); + assert_eq!( + model.token_to_piece( + &SampledToken::Content(tokens[0]), + &mut decoder, + false, + NonZeroU16::new(1), + )?, + "hello" + ); Ok(()) } @@ -315,19 +261,32 @@ fn token_to_piece_decodes_tool_call_variant(fixture: &LlamaFixture<'_>) -> Resul n_batch = 128, n_ubatch = 64, )] -fn token_to_piece_decodes_undeterminable_variant(fixture: &LlamaFixture<'_>) -> Result<()> { +fn token_to_piece_decodes_a_token_alike_in_every_non_content_section( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let model = fixture.model; - let mut decoder = encoding_rs::UTF_8.new_decoder(); - let tokens = model.str_to_token("hi", AddBos::Never, ParseSpecialTokens::Always)?; + let hi_token = single_token(model, "hi")?; - let piece = model.token_to_piece( - &SampledToken::Undeterminable(tokens[0]), - &mut decoder, - true, - None, - )?; - - assert_ne!(piece, ""); + assert_eq!( + [ + SampledToken::Reasoning(hi_token), + SampledToken::ToolCall(hi_token), + SampledToken::Undeterminable(hi_token), + ] + .map(|sampled_token| { + model.token_to_piece( + &sampled_token, + &mut encoding_rs::UTF_8.new_decoder(), + true, + None, + ) + }), + [ + Ok("hi".to_owned()), + Ok("hi".to_owned()), + Ok("hi".to_owned()) + ] + ); Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/dry_sampler.rs b/llama-cpp-bindings-tests/tests/dry_sampler.rs index 01d4d36cc..01c1a430c 100644 --- a/llama-cpp-bindings-tests/tests/dry_sampler.rs +++ b/llama-cpp-bindings-tests/tests/dry_sampler.rs @@ -1,5 +1,7 @@ use std::ffi::CString; +use anyhow::Result; + use llama_cpp_bindings::error::sampling_error::SamplingError; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_test_harness::llama_fixture::LlamaFixture; @@ -13,9 +15,14 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 2048, n_ubatch = 512, )] -fn building_a_dry_sampler_with_sequence_breakers_does_not_abort(fixture: &LlamaFixture<'_>) { +fn building_a_dry_sampler_with_sequence_breakers_succeeds( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let breakers: Vec<&[u8]> = vec![b"\n", b"\t"]; - let _sampler = LlamaSampler::dry(fixture.model, 1.5, 2.0, 128, 2, &breakers); + + LlamaSampler::dry(fixture.model, 1.5, 2.0, 128, 2, &breakers)?; + + Ok(()) } #[llama_test( @@ -26,7 +33,7 @@ fn building_a_dry_sampler_with_sequence_breakers_does_not_abort(fixture: &LlamaF n_batch = 2048, n_ubatch = 512, )] -fn dry_sampler_with_null_byte_in_seq_breakers_returns_error(fixture: &LlamaFixture<'_>) { +fn dry_sampler_refuses_a_sequence_breaker_with_a_nul_byte(fixture: &LlamaFixture<'_>) { let breakers: Vec<&[u8]> = vec![b"hello\0world"]; let result = LlamaSampler::dry(fixture.model, 1.5, 2.0, 128, 2, breakers); @@ -38,18 +45,3 @@ fn dry_sampler_with_null_byte_in_seq_breakers_returns_error(fixture: &LlamaFixtu }) ); } - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn building_a_dry_sampler_with_a_single_sequence_breaker_does_not_abort( - fixture: &LlamaFixture<'_>, -) { - let breakers: Vec<&[u8]> = vec![b"\n"]; - let _sampler = LlamaSampler::dry(fixture.model, 1.5, 2.0, 128, 2, &breakers); -} diff --git a/llama-cpp-bindings-tests/tests/embedding_availability.rs b/llama-cpp-bindings-tests/tests/embedding_availability.rs index a714d89b9..9d78323fa 100644 --- a/llama-cpp-bindings-tests/tests/embedding_availability.rs +++ b/llama-cpp-bindings-tests/tests/embedding_availability.rs @@ -41,3 +41,20 @@ fn token_embeddings_need_a_context_without_pooling(fixture: &LlamaFixture<'_>) { }) ); } + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, + embeddings = true, +)] +fn disabling_embeddings_makes_token_embeddings_unavailable(fixture: &LlamaFixture<'_>) { + let mut context = context_after_decoding_text(fixture, LlamaPoolingType::None); + + context.set_embeddings(false); + + assert_eq!(context.embeddings_ith(-1), Err(EmbeddingsError::NotEnabled)); +} diff --git a/llama-cpp-bindings-tests/tests/embedding_generation.rs b/llama-cpp-bindings-tests/tests/embedding_generation.rs index 619640ac4..47b67be6c 100644 --- a/llama-cpp-bindings-tests/tests/embedding_generation.rs +++ b/llama-cpp-bindings-tests/tests/embedding_generation.rs @@ -8,7 +8,6 @@ use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; -use llama_cpp_bindings_tests::normalize::normalize; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -22,7 +21,9 @@ use llama_cpp_test_harness_macros::llama_test; n_threads_batch = 8, embeddings = true, )] -fn embedding_generation_produces_vectors(fixture: &LlamaFixture<'_>) -> Result<()> { +fn an_embedding_pass_records_its_prompt_tokens_and_no_completion_tokens( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let model = fixture.model; let mut ctx = LlamaContext::from_model( @@ -38,42 +39,13 @@ fn embedding_generation_produces_vectors(fixture: &LlamaFixture<'_>) -> Result<( .context(format!("failed to tokenize {prompt}"))?; let prompt_token_count = u64::try_from(tokens.len())?; - let n_ctx = usize::try_from(ctx.n_ctx())?; - assert!(tokens.len() <= n_ctx, "prompt exceeds context window size"); - let mut classifier = SampledTokenClassifier::undeterminable(model); let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; - assert_eq!(classifier.pending_prompt_tokens(), prompt_token_count); - assert_eq!(classifier.usage().prompt_tokens, 0); - - ctx.memory_clear(); ctx.decode(&batch) .with_context(|| "llama_decode() failed")?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let embedding = ctx - .embeddings_seq_ith(0) - .with_context(|| "failed to get embeddings")?; - let normalized = normalize(embedding); - - assert!( - !normalized.is_empty(), - "embedding should have at least one dimension" - ); - - let magnitude: f32 = normalized - .iter() - .map(|value| value * value) - .sum::() - .sqrt(); - assert!( - (magnitude - 1.0).abs() < 0.01, - "normalized embedding magnitude should be approximately 1.0, got {magnitude}" - ); + classifier.commit_prompt_tokens(); let usage = classifier.into_usage(); assert_eq!(usage.prompt_tokens, prompt_token_count); @@ -82,34 +54,6 @@ fn embedding_generation_produces_vectors(fixture: &LlamaFixture<'_>) -> Result<( Ok(()) } -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - embeddings = true, -)] -fn an_embeddings_context_decodes_a_prompt(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - - let result = context.decode(&batch); - - assert!(result.is_ok()); - - Ok(()) -} - #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), n_gpu_layers = 999, diff --git a/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs b/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs index d1e4f2a4e..a2626e4b0 100644 --- a/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs +++ b/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs @@ -18,7 +18,9 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 2048, n_ubatch = 512, )] -fn embeddings_ith_returns_error_when_embeddings_disabled(fixture: &LlamaFixture<'_>) -> Result<()> { +fn token_embedding_reports_embeddings_not_enabled_on_a_context_without_embeddings( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let context = LlamaContext::from_model( fixture.model, fixture.backend, @@ -38,7 +40,7 @@ fn embeddings_ith_returns_error_when_embeddings_disabled(fixture: &LlamaFixture< n_batch = 2048, n_ubatch = 512, )] -fn embeddings_seq_ith_returns_error_when_embeddings_disabled( +fn sequence_embedding_reports_embeddings_not_enabled_on_a_context_without_embeddings( fixture: &LlamaFixture<'_>, ) -> Result<()> { let context = LlamaContext::from_model( @@ -64,7 +66,7 @@ fn embeddings_seq_ith_returns_error_when_embeddings_disabled( n_ubatch = 512, embeddings = true, )] -fn embeddings_seq_ith_returns_null_embedding_error_for_invalid_seq( +fn sequence_embedding_of_a_sequence_missing_from_the_decoded_batch_is_unavailable( fixture: &LlamaFixture<'_>, ) -> Result<()> { let mut context = LlamaContext::from_model( @@ -100,7 +102,7 @@ fn embeddings_seq_ith_returns_null_embedding_error_for_invalid_seq( n_ubatch = 512, embeddings = true, )] -fn embeddings_ith_returns_null_embedding_error_for_non_embedding_token( +fn token_embedding_of_an_index_without_a_decoded_output_reports_logits_not_enabled( fixture: &LlamaFixture<'_>, ) -> Result<()> { let context = LlamaContext::from_model( diff --git a/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs b/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs index 9feb62253..e263aef76 100644 --- a/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs +++ b/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs @@ -20,9 +20,7 @@ use llama_cpp_test_harness_macros::llama_test; n_ubatch = 64, embeddings = true, )] -fn encoding_more_tokens_than_the_micro_batch_returns_an_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { +fn encoding_more_tokens_than_the_micro_batch_is_refused(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; let tokens = fixture.model.str_to_token( &"hello ".repeat(100), @@ -53,7 +51,7 @@ fn encoding_more_tokens_than_the_micro_batch_returns_an_error( n_ubatch = 64, embeddings = true, )] -fn encoding_reports_a_token_llama_cpp_cannot_add_to_its_batch( +fn encoding_reports_an_exception_thrown_while_transferring_the_batch( fixture: &LlamaFixture<'_>, ) -> Result<()> { let mut context = fixture.build_context()?; diff --git a/llama-cpp-bindings-tests/tests/encoder_requirement.rs b/llama-cpp-bindings-tests/tests/encoder_requirement.rs index 8139f97f2..a56834fa6 100644 --- a/llama-cpp-bindings-tests/tests/encoder_requirement.rs +++ b/llama-cpp-bindings-tests/tests/encoder_requirement.rs @@ -17,7 +17,9 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 2048, n_ubatch = 512, )] -fn encode_on_non_encoder_model_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { +fn encoding_with_a_decoder_only_model_reports_it_has_no_encoder( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let mut context = LlamaContext::from_model( fixture.model, fixture.backend, diff --git a/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs b/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs index 633a4fb20..ff2bdebd2 100644 --- a/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs +++ b/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs @@ -37,87 +37,19 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 128, n_ubatch = 64, )] -fn is_eog_token_classifies_reasoning_variant(fixture: &LlamaFixture<'_>) -> Result<()> { +fn the_eos_token_ends_generation_in_every_section(fixture: &LlamaFixture<'_>) -> Result<()> { let model = fixture.model; let eos = model.token_eos().context("model declares an EOS token")?; - assert!(model.is_eog_token(&SampledToken::Reasoning(eos))); - Ok(()) -} -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn is_eog_token_classifies_tool_call_variant(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let eos = model.token_eos().context("model declares an EOS token")?; - assert!(model.is_eog_token(&SampledToken::ToolCall(eos))); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn is_eog_token_classifies_undeterminable_variant(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let eos = model.token_eos().context("model declares an EOS token")?; - assert!(model.is_eog_token(&SampledToken::Undeterminable(eos))); + assert_eq!( + [ + SampledToken::Content(eos), + SampledToken::Reasoning(eos), + SampledToken::ToolCall(eos), + SampledToken::Undeterminable(eos), + ] + .map(|sampled_token| model.is_eog_token(&sampled_token)), + [true, true, true, true] + ); Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/grammar_sampler.rs b/llama-cpp-bindings-tests/tests/grammar_sampler.rs index 7cb21fc6d..3c76529e4 100644 --- a/llama-cpp-bindings-tests/tests/grammar_sampler.rs +++ b/llama-cpp-bindings-tests/tests/grammar_sampler.rs @@ -1,8 +1,6 @@ use std::ffi::CString; -use anyhow::Context as _; use anyhow::Result; -use anyhow::bail; use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::context::LlamaContext; @@ -16,6 +14,9 @@ use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; +use llama_cpp_bindings_tests::logits_left_by::logits_left_by; +use llama_cpp_bindings_tests::single_token::single_token; +use llama_cpp_bindings_tests::yes_or_no_prefixes::YES_OR_NO_PREFIXES; use llama_cpp_fault_injection::armed_fault::ArmedFault; use llama_cpp_gbnf::gbnf_validation_error::GbnfValidationError; use llama_cpp_test_harness::llama_fixture::LlamaFixture; @@ -53,7 +54,9 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 512, n_ubatch = 128, )] -fn grammar_sampler_constrains_output_to_yes_or_no(fixture: &LlamaFixture<'_>) -> Result<()> { +fn grammar_sampler_constrains_the_first_token_to_a_prefix_of_yes_or_no( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let model = fixture.model; let mut context = LlamaContext::from_model( model, @@ -94,22 +97,10 @@ fn grammar_sampler_constrains_output_to_yes_or_no(fixture: &LlamaFixture<'_>) -> ); let piece = outcome.piece.raw(); - let first_char = piece - .chars() - .next() - .context("piece should have at least one character")? - .to_lowercase() - .next() - .context("lowercase iterator should yield a character")?; assert!( - first_char == 'y' || first_char == 'n', - "Grammar should constrain first token to start with y/n, got: '{piece}'" - ); - assert_eq!( - classifier.usage().completion_tokens(), - 1, - "exactly one completion token sampled" + YES_OR_NO_PREFIXES.contains(&piece.to_lowercase().as_str()), + "the grammar must constrain the first token to a prefix of yes or no, got: '{piece}'" ); Ok(()) @@ -147,7 +138,7 @@ fn grammar_sampler_constrains_output_to_yes_or_no(fixture: &LlamaFixture<'_>) -> n_batch = 512, n_ubatch = 128, )] -fn sample_with_grammar_produces_constrained_output_in_loop( +fn grammar_constrained_chat_answer_is_yes_or_no_content_that_ends_generation( fixture: &LlamaFixture<'_>, ) -> Result<()> { let model = fixture.model; @@ -216,10 +207,21 @@ fn sample_with_grammar_produces_constrained_output_in_loop( n_batch = 2048, n_ubatch = 512, )] -fn grammar_returns_sampler_for_valid_grammar(fixture: &LlamaFixture<'_>) { - let sampler = LlamaSampler::grammar(fixture.model, "root ::= \"hello\"", "root"); +fn a_grammar_sampler_masks_tokens_that_cannot_start_the_grammar( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let sampler = LlamaSampler::grammar(model, "root ::= \"hello\"", "root")?; + + assert_eq!( + logits_left_by( + &sampler, + &[single_token(model, "hello")?, single_token(model, "yes")?] + )?, + [0.0, f32::NEG_INFINITY] + ); - assert!(sampler.is_ok()); + Ok(()) } #[llama_test( @@ -234,11 +236,7 @@ fn qwen35_grammar_with_a_token_reference_constrains_the_first_token( fixture: &LlamaFixture<'_>, ) -> Result<()> { let model = fixture.model; - let [think_token] = - model.str_to_token("", AddBos::Never, ParseSpecialTokens::Always)?[..] - else { - bail!(" must be a single token"); - }; + let think_token = single_token(model, "")?; let mut context = LlamaContext::from_model( model, fixture.backend, diff --git a/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs b/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs index a9ee2c57d..3f8f531ff 100644 --- a/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs +++ b/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs @@ -88,6 +88,32 @@ fn hidden_state_after( .to_vec()) } +fn context_with_a_decoded_fork<'fixture>( + fixture: &'fixture LlamaFixture<'_>, + state: &[LlamaToken], + branch: &[LlamaToken], +) -> Result> { + let mut context = hidden_state_context(fixture)?; + let mut state_batch = LlamaBatch::new(ContextLimits::of(&context)); + + add_tokens(&mut state_batch, state, 0, 0, false)?; + context.decode(&state_batch)?; + context.memory_seq_cp(0, 1, None, None)?; + + let mut branch_batch = LlamaBatch::new(ContextLimits::of(&context)); + + add_tokens( + &mut branch_batch, + branch, + i32::try_from(state.len())?, + 1, + true, + )?; + context.decode(&branch_batch)?; + + Ok(context) +} + fn decode_tokens( context: &mut LlamaContext<'_>, tokens: &[LlamaToken], @@ -249,35 +275,37 @@ fn nextn_embeddings_are_unavailable_until_enabled(fixture: &LlamaFixture<'_>) -> n_ubatch = 512, n_seq_max = 2, )] -fn a_released_fork_leaves_its_shared_state_as_an_independent_decode_would( +fn a_forked_branch_has_the_hidden_state_of_an_independent_decode( fixture: &LlamaFixture<'_>, ) -> Result<()> { let state = tokenize(fixture, STATE)?; let first_branch = tokenize(fixture, FIRST_BRANCH)?; - let second_branch = tokenize(fixture, SECOND_BRANCH)?; - let branch_position = i32::try_from(state.len())?; - - let mut context = hidden_state_context(fixture)?; - let mut state_batch = LlamaBatch::new(ContextLimits::of(&context)); + let context = context_with_a_decoded_fork(fixture, &state, &first_branch)?; - add_tokens(&mut state_batch, &state, 0, 0, false)?; - context.decode(&state_batch)?; - context.memory_seq_cp(0, 1, None, None)?; - - let mut first_branch_batch = LlamaBatch::new(ContextLimits::of(&context)); + assert_eq!( + context.nextn_embeddings_ith(i32::try_from(first_branch.len())? - 1)?, + hidden_state_after(fixture, &state, &first_branch)?.as_slice(), + ); - add_tokens( - &mut first_branch_batch, - &first_branch, - branch_position, - 1, - true, - )?; - context.decode(&first_branch_batch)?; + Ok(()) +} - let first_branch_hidden_state = context - .nextn_embeddings_ith(first_branch_batch.n_tokens() - 1)? - .to_vec(); +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, + n_seq_max = 2, +)] +fn a_released_fork_leaves_its_shared_state_as_an_independent_decode_would( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let state = tokenize(fixture, STATE)?; + let second_branch = tokenize(fixture, SECOND_BRANCH)?; + let mut context = + context_with_a_decoded_fork(fixture, &state, &tokenize(fixture, FIRST_BRANCH)?)?; context.memory_seq_rm(Some(1), None, None)?; @@ -286,16 +314,12 @@ fn a_released_fork_leaves_its_shared_state_as_an_independent_decode_would( add_tokens( &mut second_branch_batch, &second_branch, - branch_position, + i32::try_from(state.len())?, 0, true, )?; context.decode(&second_branch_batch)?; - assert_eq!( - first_branch_hidden_state, - hidden_state_after(fixture, &state, &first_branch)?, - ); assert_eq!( context.nextn_embeddings_ith(second_branch_batch.n_tokens() - 1)?, hidden_state_after(fixture, &state, &second_branch)?.as_slice(), @@ -339,31 +363,6 @@ fn final_hidden_states_after_enabling_embeddings_for_a_row_equal_those_of_an_emb Ok(()) } -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, -)] -fn disabling_embeddings_makes_token_embeddings_unavailable( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let state = tokenize(fixture, STATE)?; - let mut context = final_hidden_state_context(fixture, true)?; - let last_index = decode_tokens(&mut context, &state, 0, true)?; - - context.set_embeddings(false); - - assert_eq!( - context.embeddings_ith(last_index), - Err(EmbeddingsError::NotEnabled) - ); - - Ok(()) -} - #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), n_gpu_layers = 999, diff --git a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs index 0d6400fd9..c6196359b 100644 --- a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs +++ b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs @@ -1,5 +1,6 @@ use anyhow::Context as _; use anyhow::Result; +use serde_json::Value; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; @@ -10,9 +11,12 @@ use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; +const MAX_ANSWER_TOKENS: i32 = 64; + #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -45,7 +49,9 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 512, n_ubatch = 128, )] -fn json_schema_grammar_sampler_constrains_output_to_json(fixture: &LlamaFixture<'_>) -> Result<()> { +fn json_schema_grammar_sampler_constrains_the_answer_to_the_schema( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let model = fixture.model; let mut context = LlamaContext::from_model( model, @@ -56,10 +62,11 @@ fn json_schema_grammar_sampler_constrains_output_to_json(fixture: &LlamaFixture< let prompt = "<|im_start|>user\nWhat is 2+2? Respond with a JSON object.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + let mut classifier = SampledTokenClassifier::undeterminable(model); - batch.add_sequence(&tokens, 0, false)?; - + classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; context.decode(&batch)?; + classifier.commit_prompt_tokens(); let grammar_str = json_schema_to_grammar( r#"{"type": "object", "properties": {"answer": {"type": "string"}}, "required": ["answer"]}"#, @@ -71,35 +78,29 @@ fn json_schema_grammar_sampler_constrains_output_to_json(fixture: &LlamaFixture< LlamaSampler::greedy()?, ])?; - let mut classifier = SampledTokenClassifier::undeterminable(model); - let mut outcomes = Vec::new(); - let turn = classifier.sample(&mut sampler, &context, batch.n_tokens() - 1, &mut outcomes)?; - classifier.finish(&mut outcomes)?; - - assert_eq!( - outcomes.len(), - 1, - "expected one finalised outcome after finishing" - ); - let outcome = &outcomes[0]; + let initial_position = batch.n_tokens(); + let outcome = ClassifySampleLoop { + classifier: &mut classifier, + sampler: &mut sampler, + context: &mut context, + batch: &mut batch, + initial_position, + max_generated_tokens: MAX_ANSWER_TOKENS, + } + .run(); - let raw_as_sampled = SampledToken::Content(turn.token); assert!( - !model.is_eog_token(&raw_as_sampled), - "Grammar sampler should not allow EOS as first token" + outcome.eog_seen, + "the answer must end once the schema is satisfied; outcome={outcome:?}" ); - let piece = outcome.piece.raw(); + let answer = outcome.generated.text().parse::()?; + let answer_object = answer + .as_object() + .context("the schema requires the answer to be a JSON object")?; - assert!( - piece.starts_with('{'), - "JSON schema grammar should constrain first token to start with '{{', got: '{piece}'" - ); - assert_eq!( - classifier.usage().completion_tokens(), - 1, - "exactly one completion token sampled" - ); + assert_eq!(answer_object.keys().collect::>(), ["answer"]); + assert!(answer_object["answer"].is_string(), "{answer}"); Ok(()) } @@ -199,12 +200,12 @@ fn an_llguidance_json_schema_constrains_the_answer_to_the_schema( } let parsed = serde_json::Deserializer::from_str(&generated) - .into_iter::() + .into_iter::() .next() .context("model produced no JSON value")??; - assert!(parsed.get("city").is_some()); - assert!(parsed.get("temperature").is_some()); + assert!(parsed["city"].is_string(), "{parsed}"); + assert!(parsed["temperature"].is_number(), "{parsed}"); Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs b/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs index 40092bd0d..e8a873618 100644 --- a/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs +++ b/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs @@ -3,9 +3,9 @@ use std::ffi::CString; use anyhow::Result; use llama_cpp_bindings::error::grammar_error::GrammarError; -use llama_cpp_bindings::model::add_bos::AddBos; -use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings_tests::logits_left_by::logits_left_by; +use llama_cpp_bindings_tests::single_token::single_token; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -17,12 +17,22 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 2048, n_ubatch = 512, )] -fn grammar_lazy_returns_sampler_for_valid_grammar_with_patterns(fixture: &LlamaFixture<'_>) { +fn a_lazy_grammar_awaiting_its_trigger_pattern_leaves_candidates_unconstrained( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; let patterns = vec!["\\{.*".to_owned()]; - let sampler = - LlamaSampler::grammar_lazy(fixture.model, "root ::= \"hello\"", "root", &patterns, &[]); + let sampler = LlamaSampler::grammar_lazy(model, "root ::= \"hello\"", "root", &patterns, &[])?; + + assert_eq!( + logits_left_by( + &sampler, + &[single_token(model, "hello")?, single_token(model, "yes")?] + )?, + [0.0, 0.0] + ); - assert!(sampler.is_ok()); + Ok(()) } #[llama_test( @@ -33,7 +43,7 @@ fn grammar_lazy_returns_sampler_for_valid_grammar_with_patterns(fixture: &LlamaF n_batch = 2048, n_ubatch = 512, )] -fn grammar_lazy_with_root_not_found_returns_error(fixture: &LlamaFixture<'_>) { +fn grammar_lazy_without_its_root_reports_the_missing_root(fixture: &LlamaFixture<'_>) { let patterns = vec!["function".to_owned()]; let result = LlamaSampler::grammar_lazy(fixture.model, "expr ::= \"hello\"", "root", &patterns, &[]); @@ -49,16 +59,16 @@ fn grammar_lazy_with_root_not_found_returns_error(fixture: &LlamaFixture<'_>) { n_batch = 2048, n_ubatch = 512, )] -fn grammar_lazy_with_null_byte_in_pattern_returns_error(fixture: &LlamaFixture<'_>) { +fn grammar_lazy_refuses_a_trigger_pattern_with_a_nul_byte(fixture: &LlamaFixture<'_>) { let patterns = vec!["hel\0lo".to_owned()]; let result = LlamaSampler::grammar_lazy(fixture.model, "root ::= \"hello\"", "root", &patterns, &[]); assert_eq!( result.err(), - Some(GrammarError::TriggerPatternContainsNul( - CString::new("hel\0lo").expect_err("the pattern carries a nul byte") - )) + CString::new("hel\0lo") + .err() + .map(GrammarError::TriggerPatternContainsNul) ); } @@ -70,30 +80,24 @@ fn grammar_lazy_with_null_byte_in_pattern_returns_error(fixture: &LlamaFixture<' n_batch = 2048, n_ubatch = 512, )] -fn grammar_lazy_returns_sampler_for_valid_grammar_with_trigger_tokens( +fn a_lazy_grammar_awaiting_its_trigger_token_leaves_candidates_unconstrained( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let trigger_tokens = - fixture - .model - .str_to_token("{", AddBos::Never, ParseSpecialTokens::Always)?; - - assert!( - !trigger_tokens.is_empty(), - "the tokenizer must produce at least one trigger token" - ); - + let model = fixture.model; let sampler = LlamaSampler::grammar_lazy( - fixture.model, + model, "root ::= \"hello\"", "root", &[], - &trigger_tokens, - ); + &[single_token(model, "{")?], + )?; - assert!( - sampler.is_ok(), - "trigger tokens alone must be enough to build a lazy grammar sampler" + assert_eq!( + logits_left_by( + &sampler, + &[single_token(model, "hello")?, single_token(model, "yes")?] + )?, + [0.0, 0.0] ); Ok(()) @@ -107,15 +111,15 @@ fn grammar_lazy_returns_sampler_for_valid_grammar_with_trigger_tokens( n_batch = 2048, n_ubatch = 512, )] -fn grammar_lazy_with_null_byte_in_grammar_returns_error(fixture: &LlamaFixture<'_>) { +fn grammar_lazy_refuses_a_grammar_with_a_nul_byte(fixture: &LlamaFixture<'_>) { let result = LlamaSampler::grammar_lazy(fixture.model, "root ::= \"hel\0lo\"", "root", &[], &[]); assert_eq!( result.err(), - Some(GrammarError::GrammarContainsNul( - CString::new("root ::= \"hel\0lo\"").expect_err("the grammar carries a nul byte") - )) + CString::new("root ::= \"hel\0lo\"") + .err() + .map(GrammarError::GrammarContainsNul) ); } diff --git a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs index 126fddb82..c2c43ca6c 100644 --- a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs +++ b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs @@ -2,6 +2,7 @@ use std::ffi::CStr; use anyhow::Context as _; use anyhow::Result; +use anyhow::bail; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; @@ -16,8 +17,12 @@ use llama_cpp_bindings::model::LlamaModel; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::params::LlamaModelParams; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_bindings_tests::logits_left_by::logits_left_by; +use llama_cpp_bindings_tests::single_token::single_token; +use llama_cpp_bindings_tests::yes_or_no_prefixes::YES_OR_NO_PREFIXES; use llama_cpp_fault_injection::armed_fault::ArmedFault; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -26,26 +31,21 @@ const JSON_SCHEMA: &str = r#"{"type":"object","properties":{"answer":{"type":"string"}},"required":["answer"]}"#; const REGEX_GRAMMAR: &str = r"yes|no"; +const LARK_GRAMMAR: &str = r#"start: "yes" | "no""#; + fn out_of_vocabulary_failure() -> String { GrammarRuntimeError::InternalParserError { message: "token id 2147483646 out of range".to_owned(), } .to_string() } -const LARK_GRAMMAR: &str = r#"start: "yes" | "no""#; - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn llguidance_method_creates_sampler(fixture: &LlamaFixture<'_>) { - let result = LlamaSampler::llguidance(fixture.model, "regex", r"yes|no"); - assert!(result.is_ok()); +fn yes_no_and_hello(model: &LlamaModel) -> Result> { + Ok(vec![ + single_token(model, "yes")?, + single_token(model, "no")?, + single_token(model, "hello")?, + ]) } #[llama_test( @@ -80,14 +80,31 @@ fn llguidance_method_creates_sampler(fixture: &LlamaFixture<'_>) { n_batch = 512, n_ubatch = 128, )] -fn creates_sampler_with_valid_json_schema(fixture: &LlamaFixture<'_>) -> Result<()> { - let sampler = create_llg_sampler(fixture.model, "json", JSON_SCHEMA)?; +fn a_json_schema_sampler_masks_tokens_that_cannot_start_the_schema( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let sampler = create_llg_sampler(model, "json", JSON_SCHEMA)?; - assert!(!sampler.as_ptr().is_null()); + assert_eq!( + logits_left_by( + &sampler, + &[single_token(model, "{")?, single_token(model, "hello")?] + )?, + [0.0, f32::NEG_INFINITY] + ); Ok(()) } +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -120,10 +137,16 @@ fn creates_sampler_with_valid_json_schema(fixture: &LlamaFixture<'_>) -> Result< n_batch = 512, n_ubatch = 128, )] -fn creates_sampler_with_valid_regex_grammar(fixture: &LlamaFixture<'_>) -> Result<()> { - let sampler = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; +fn a_regex_sampler_masks_tokens_that_cannot_start_a_match( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let sampler = LlamaSampler::llguidance(model, "regex", REGEX_GRAMMAR)?; - assert!(!sampler.as_ptr().is_null()); + assert_eq!( + logits_left_by(&sampler, &yes_no_and_hello(model)?)?, + [0.0, 0.0, f32::NEG_INFINITY] + ); Ok(()) } @@ -160,10 +183,16 @@ fn creates_sampler_with_valid_regex_grammar(fixture: &LlamaFixture<'_>) -> Resul n_batch = 512, n_ubatch = 128, )] -fn creates_sampler_with_valid_lark_grammar(fixture: &LlamaFixture<'_>) -> Result<()> { - let sampler = create_llg_sampler(fixture.model, "lark", LARK_GRAMMAR)?; +fn a_lark_sampler_masks_tokens_that_cannot_start_the_grammar( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let sampler = create_llg_sampler(model, "lark", LARK_GRAMMAR)?; - assert!(!sampler.as_ptr().is_null()); + assert_eq!( + logits_left_by(&sampler, &yes_no_and_hello(model)?)?, + [0.0, 0.0, f32::NEG_INFINITY] + ); Ok(()) } @@ -200,10 +229,13 @@ fn creates_sampler_with_valid_lark_grammar(fixture: &LlamaFixture<'_>) -> Result n_batch = 512, n_ubatch = 128, )] -fn returns_error_for_unknown_grammar_kind(fixture: &LlamaFixture<'_>) { - let result = create_llg_sampler(fixture.model, "not_a_real_kind", "anything"); - - assert!(result.is_err()); +fn an_unknown_grammar_kind_is_reported_as_an_invalid_grammar(fixture: &LlamaFixture<'_>) { + assert_eq!( + create_llg_sampler(fixture.model, "not_a_real_kind", "anything").err(), + Some(GrammarError::LlguidanceGrammarInvalid { + message: "unknown constraint type: not_a_real_kind".to_owned(), + }) + ); } #[llama_test( @@ -238,10 +270,13 @@ fn returns_error_for_unknown_grammar_kind(fixture: &LlamaFixture<'_>) { n_batch = 512, n_ubatch = 128, )] -fn returns_error_for_malformed_json_schema(fixture: &LlamaFixture<'_>) { - let result = create_llg_sampler(fixture.model, "json", "{this is not valid json"); - - assert!(result.is_err()); +fn a_json_schema_that_is_not_json_is_reported_as_an_invalid_grammar(fixture: &LlamaFixture<'_>) { + assert_eq!( + create_llg_sampler(fixture.model, "json", "{this is not valid json").err(), + Some(GrammarError::LlguidanceGrammarInvalid { + message: "key must be a string at line 1 column 2".to_owned(), + }) + ); } #[llama_test( @@ -276,10 +311,14 @@ fn returns_error_for_malformed_json_schema(fixture: &LlamaFixture<'_>) { n_batch = 512, n_ubatch = 128, )] -fn returns_error_for_malformed_regex(fixture: &LlamaFixture<'_>) { - let result = create_llg_sampler(fixture.model, "regex", "[invalid"); +fn a_malformed_regex_leaves_llguidance_without_a_parser(fixture: &LlamaFixture<'_>) -> Result<()> { + let Err(GrammarError::LlguidanceParserUnavailable { .. }) = + create_llg_sampler(fixture.model, "regex", "[invalid") + else { + bail!("a regex with an unclosed character class must leave llguidance without a parser"); + }; - assert!(result.is_err()); + Ok(()) } #[llama_test( @@ -358,14 +397,20 @@ fn name_callback_returns_llguidance(fixture: &LlamaFixture<'_>) -> Result<()> { n_batch = 512, n_ubatch = 128, )] -fn clone_via_ffi_creates_independent_sampler(fixture: &LlamaFixture<'_>) -> Result<()> { - let sampler = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; - - let cloned = unsafe { llama_cpp_bindings_sys::llama_sampler_clone(sampler.as_ptr()) }; - - assert!(!cloned.is_null()); +fn a_sampler_cloned_through_ffi_keeps_its_own_grammar_state( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let mut sampler = create_llg_sampler(model, "regex", REGEX_GRAMMAR)?; + let mut cloned = unsafe { + LlamaSampler::from_raw( + llama_cpp_bindings_sys::llama_sampler_clone(sampler.as_ptr()), + "llguidance", + ) + }?; - unsafe { llama_cpp_bindings_sys::llama_sampler_free(cloned) }; + assert_eq!(cloned.accept(single_token(model, "yes")?), Ok(())); + assert_eq!(sampler.accept(single_token(model, "no")?), Ok(())); Ok(()) } @@ -402,7 +447,7 @@ fn clone_via_ffi_creates_independent_sampler(fixture: &LlamaFixture<'_>) -> Resu n_batch = 512, n_ubatch = 128, )] -fn samples_token_constrained_by_grammar(fixture: &LlamaFixture<'_>) -> Result<()> { +fn llguidance_chain_samples_a_prefix_of_yes_or_no(fixture: &LlamaFixture<'_>) -> Result<()> { let model = fixture.model; let backend = fixture.backend; let mut context = LlamaContext::from_model( @@ -421,9 +466,16 @@ fn samples_token_constrained_by_grammar(fixture: &LlamaFixture<'_>) -> Result<() let mut chain = LlamaSampler::chain([llg_sampler, LlamaSampler::greedy()?])?; let token = chain.sample(&context, batch.n_tokens() - 1)?; + let piece = model.token_to_piece( + &SampledToken::Content(token), + &mut encoding_rs::UTF_8.new_decoder(), + false, + None, + )?; + assert!( - token.0 >= 0, - "grammar-constrained sampling must yield a valid token id without the grammar rejecting it" + YES_OR_NO_PREFIXES.contains(&piece.as_str()), + "the grammar must constrain the sampled token to a prefix of yes or no, got: '{piece}'" ); Ok(()) @@ -544,69 +596,18 @@ fn accept_maps_an_out_of_vocabulary_token_to_grammar_callback_failure( n_batch = 512, n_ubatch = 128, )] -fn approximate_tok_env_drives_consistent_grammar_constraint( +fn samplers_sharing_the_cached_tokenizer_environment_mask_alike( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let first = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; - let second = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; - - assert!(!first.as_ptr().is_null()); - assert!(!second.as_ptr().is_null()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn llguidance_chain_samples_a_valid_token(fixture: &LlamaFixture<'_>) -> Result<()> { let model = fixture.model; - let backend = fixture.backend; - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let tokens = model.str_to_token("Answer:", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; + let candidates = yes_no_and_hello(model)?; + let first = create_llg_sampler(model, "regex", REGEX_GRAMMAR)?; + let second = create_llg_sampler(model, "regex", REGEX_GRAMMAR)?; - let llg_sampler = create_llg_sampler(model, "regex", REGEX_GRAMMAR)?; - let mut chain = LlamaSampler::chain([llg_sampler, LlamaSampler::greedy()?])?; - let token = chain.sample(&context, batch.n_tokens() - 1)?; - - assert!(token.0 >= 0); + assert_eq!( + logits_left_by(&first, &candidates)?, + logits_left_by(&second, &candidates)? + ); Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs b/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs index d314ce8d7..1dcdfe945 100644 --- a/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs +++ b/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs @@ -1,4 +1,8 @@ +use anyhow::Result; + use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_bindings_tests::logits_left_by::logits_left_by; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -10,8 +14,13 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 2048, n_ubatch = 512, )] -fn a_logit_bias_sampler_accepts_an_empty_bias_list(_fixture: &LlamaFixture<'_>) { - let result = LlamaSampler::logit_bias(0, &[]); +fn a_logit_bias_sampler_with_an_empty_bias_list_leaves_candidates_unchanged( + _fixture: &LlamaFixture<'_>, +) -> Result<()> { + assert_eq!( + logits_left_by(&LlamaSampler::logit_bias(0, &[])?, &[LlamaToken::new(0)])?, + [0.0] + ); - assert!(result.is_ok()); + Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/logits_availability.rs b/llama-cpp-bindings-tests/tests/logits_availability.rs index 74e3fc9be..bd3aff655 100644 --- a/llama-cpp-bindings-tests/tests/logits_availability.rs +++ b/llama-cpp-bindings-tests/tests/logits_availability.rs @@ -10,26 +10,6 @@ use llama_cpp_bindings_tests::context_after_decoding_text::context_after_decodin use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, - embeddings = true, -)] -fn a_pooled_embeddings_decode_produces_no_logits(fixture: &LlamaFixture<'_>) { - let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean); - - assert_eq!( - context.get_logits_ith(-1), - Err(LogitsError::PooledEmbeddingsProduceNoLogits { - pooling_type: LlamaPoolingType::Mean, - }) - ); -} - #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), n_gpu_layers = 999, diff --git a/llama-cpp-bindings-tests/tests/lora_adapters.rs b/llama-cpp-bindings-tests/tests/lora_adapters.rs index 4208a3048..7e5cf2900 100644 --- a/llama-cpp-bindings-tests/tests/lora_adapters.rs +++ b/llama-cpp-bindings-tests/tests/lora_adapters.rs @@ -1,6 +1,15 @@ +use std::env; +use std::ffi::CString; +use std::ffi::OsStr; +use std::fs; +use std::os::unix::ffi::OsStrExt; +use std::path::Path; +use std::path::PathBuf; + use anyhow::Result; use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::error::llama_lora_adapter_init_error::LlamaLoraAdapterInitError; use llama_cpp_bindings::error::llama_lora_adapters_error::LlamaLoraAdaptersError; use llama_cpp_bindings::model::lora_adapter_scale::LoraAdapterScale; use llama_cpp_fault_injection::armed_fault::ArmedFault; @@ -39,8 +48,6 @@ fn set_lora_adapters_applies_a_real_adapter(fixture: &LlamaFixture<'_>) -> Resul Ok(()) ); - assert_eq!(context.set_lora_adapters(&[]), Ok(())); - Ok(()) } @@ -58,9 +65,7 @@ fn set_lora_adapters_accepts_an_empty_adapter_set(fixture: &LlamaFixture<'_>) -> fixture.backend, (*fixture.context_params).into_llama_context_params(), )?; - let result = context.set_lora_adapters(&[]); - - assert!(result.is_ok()); + assert_eq!(context.set_lora_adapters(&[]), Ok(())); Ok(()) } @@ -98,3 +103,80 @@ fn setting_lora_adapters_reports_a_status_llama_cpp_returns( Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn lora_adapter_init_with_a_missing_file_reports_it_not_found(fixture: &LlamaFixture<'_>) { + let result = fixture + .model + .lora_adapter_init("/nonexistent/path/lora.gguf"); + assert_eq!( + result.unwrap_err(), + LlamaLoraAdapterInitError::FileNotFound(PathBuf::from("/nonexistent/path/lora.gguf")) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn lora_adapter_init_with_an_invalid_gguf_reports_it_unloadable( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let dummy_path = env::temp_dir().join("llama_test_dummy_lora.gguf"); + fs::write(&dummy_path, b"not a valid gguf")?; + + let result = fixture.model.lora_adapter_init(&dummy_path); + + assert_eq!(result.unwrap_err(), LlamaLoraAdapterInitError::Unloadable); + fs::remove_file(&dummy_path)?; + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn lora_adapter_init_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) { + let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.gguf")); + let result = fixture.model.lora_adapter_init(non_utf8_path); + + assert_eq!( + result.unwrap_err(), + LlamaLoraAdapterInitError::PathToStrError(non_utf8_path.to_path_buf()) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn lora_adapter_init_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) { + assert_eq!( + fixture + .model + .lora_adapter_init("/adapters/adapter\0.gguf") + .err(), + CString::new("/adapters/adapter\0.gguf") + .err() + .map(LlamaLoraAdapterInitError::NullError) + ); +} diff --git a/llama-cpp-bindings-tests/tests/memory_availability.rs b/llama-cpp-bindings-tests/tests/memory_availability.rs index 505f946e5..c83d28964 100644 --- a/llama-cpp-bindings-tests/tests/memory_availability.rs +++ b/llama-cpp-bindings-tests/tests/memory_availability.rs @@ -51,15 +51,35 @@ fn a_causal_embedding_context_keeps_memory(fixture: &LlamaFixture<'_>) -> Result n_threads_batch = 8, embeddings = true, )] -fn memory_operations_respect_an_embedding_context_without_memory( +fn memory_edits_without_a_memory_module_trivially_succeed( fixture: &LlamaFixture<'_>, ) -> Result<()> { let mut context = fixture.build_context()?; context.memory_clear(); - assert_eq!(context.memory_seq_rm(Some(0), None, None), Ok(())); context.memory_seq_keep(0); + assert_eq!(context.memory_seq_rm(Some(0), None, None), Ok(())); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + n_threads_batch = 8, + embeddings = true, +)] +fn memory_operations_needing_a_memory_module_report_its_absence( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + let divisor = NonZeroU8::new(2).context("2 is non-zero")?; + assert_eq!( context.memory_seq_cp(0, 1, None, None), Err(MemorySeqCpError::MemoryHandleUnavailable) @@ -68,7 +88,6 @@ fn memory_operations_respect_an_embedding_context_without_memory( context.memory_seq_add(0, None, None, 1), Err(MemorySeqAddError::MemoryHandleUnavailable) ); - let divisor = NonZeroU8::new(2).context("2 is non-zero")?; assert_eq!( context.memory_seq_div(0, None, None, divisor), Err(MemorySeqDivError::MemoryHandleUnavailable) @@ -77,10 +96,31 @@ fn memory_operations_respect_an_embedding_context_without_memory( context.memory_seq_pos_max(0), Err(MemorySeqPosMaxError::MemoryHandleUnavailable) ); - assert!(matches!( - context.memory_seq_rm(Some(u32::MAX), None, None), - Err(MemorySeqRmError::SeqIdTooLarge(_)) - )); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + n_threads_batch = 8, + embeddings = true, +)] +fn removing_a_sequence_without_a_memory_module_still_validates_its_id( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + assert_eq!( + context.memory_seq_rm(Some(u32::MAX), None, None).err(), + i32::try_from(u32::MAX) + .err() + .map(MemorySeqRmError::SeqIdTooLarge) + ); Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/memory_occupancy.rs b/llama-cpp-bindings-tests/tests/memory_occupancy.rs index b236364fb..5da38d551 100644 --- a/llama-cpp-bindings-tests/tests/memory_occupancy.rs +++ b/llama-cpp-bindings-tests/tests/memory_occupancy.rs @@ -1,9 +1,15 @@ use anyhow::Result; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings_tests::prime_memory::prime_memory; +use llama_cpp_bindings_tests::prime_memory_with::prime_memory_with; +use llama_cpp_bindings_tests::two_sequence_context::two_sequence_context; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; +const PRIMED_TEXT: &str = "Hello world"; + #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -79,12 +85,21 @@ fn memory_clear_resets_positions(fixture: &LlamaFixture<'_>) -> Result<()> { n_batch = 512, n_ubatch = 128, )] -fn memory_seq_pos_max_is_non_negative_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_pos_max_is_the_position_of_the_last_decoded_token( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let mut context = fixture.build_context()?; + let primed_tokens = + fixture + .model + .str_to_token(PRIMED_TEXT, AddBos::Always, ParseSpecialTokens::Always)?; - prime_memory(fixture, &mut context)?; + prime_memory_with(fixture, &mut context, PRIMED_TEXT)?; - assert!(context.memory_seq_pos_max(0)? >= 0); + assert_eq!( + context.memory_seq_pos_max(0)?, + i32::try_from(primed_tokens.len())? - 1 + ); Ok(()) } @@ -121,14 +136,18 @@ fn memory_seq_pos_max_is_non_negative_after_decode(fixture: &LlamaFixture<'_>) - n_batch = 512, n_ubatch = 128, )] -fn memory_seq_keep_retains_specified_sequence(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; +fn memory_seq_keep_removes_every_other_sequence(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = two_sequence_context(fixture)?; prime_memory(fixture, &mut context)?; - context.memory_seq_keep(0); + let primed_position_max = context.memory_seq_pos_max(0)?; - assert!(context.memory_seq_pos_max(0)? >= 0); + context.memory_seq_cp(0, 1, None, None)?; + context.memory_seq_keep(1); + + assert_eq!(context.memory_seq_pos_max(0)?, -1); + assert_eq!(context.memory_seq_pos_max(1)?, primed_position_max); Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs b/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs index dcf2a9456..3117e9f7a 100644 --- a/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs +++ b/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs @@ -25,7 +25,7 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 512, n_ubatch = 128, )] -fn memory_seq_add_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_add_refuses_an_mrope_model(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; prime_memory(fixture, &mut context)?; @@ -53,7 +53,7 @@ fn memory_seq_add_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> R n_batch = 512, n_ubatch = 128, )] -fn memory_seq_div_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_div_refuses_an_mrope_model(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; prime_memory(fixture, &mut context)?; @@ -74,14 +74,18 @@ fn memory_seq_div_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> R n_batch = 512, n_ubatch = 128, )] -fn memory_seq_add_succeeds_on_embedding_model(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_add_shifts_the_positions_of_a_standard_rope_model( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let mut context = fixture.build_context()?; prime_memory(fixture, &mut context)?; - let result = context.memory_seq_add(0, Some(0), None, 1); + let primed_position_max = context.memory_seq_pos_max(0)?; + + context.memory_seq_add(0, Some(0), None, 1)?; - assert!(result.is_ok()); + assert_eq!(context.memory_seq_pos_max(0)?, primed_position_max + 1); Ok(()) } @@ -94,15 +98,22 @@ fn memory_seq_add_succeeds_on_embedding_model(fixture: &LlamaFixture<'_>) -> Res n_batch = 512, n_ubatch = 128, )] -fn memory_seq_div_succeeds_on_embedding_model(fixture: &LlamaFixture<'_>) -> Result<()> { +fn memory_seq_div_divides_the_positions_of_a_standard_rope_model( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let mut context = fixture.build_context()?; prime_memory(fixture, &mut context)?; + let primed_position_max = context.memory_seq_pos_max(0)?; let divisor = NonZeroU8::new(2).context("2 is non-zero")?; - let result = context.memory_seq_div(0, Some(0), None, divisor); - assert!(result.is_ok()); + context.memory_seq_div(0, Some(0), None, divisor)?; + + assert_eq!( + context.memory_seq_pos_max(0)?, + primed_position_max / i32::from(divisor.get()) + ); Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/memory_priming.rs b/llama-cpp-bindings-tests/tests/memory_priming.rs index 2d786870d..b38e38a0b 100644 --- a/llama-cpp-bindings-tests/tests/memory_priming.rs +++ b/llama-cpp-bindings-tests/tests/memory_priming.rs @@ -1,7 +1,12 @@ +use std::ffi::CString; + use anyhow::Result; +use anyhow::bail; use llama_cpp_bindings::batch_add_error::BatchAddError; +use llama_cpp_bindings::error::decode_error::DecodeError; use llama_cpp_bindings::error::string_to_token_error::StringToTokenError; +use llama_cpp_bindings_tests::prime_memory::prime_memory; use llama_cpp_bindings_tests::prime_memory_with::prime_memory_with; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -14,14 +19,33 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 256, n_ubatch = 64, )] -fn prime_memory_surfaces_each_underlying_error(fixture: &LlamaFixture<'_>) -> Result<()> { +fn priming_memory_surfaces_a_tokenization_error(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; + let Err(failure) = prime_memory_with(fixture, &mut context, "Hello\0world") else { + bail!("an interior NUL byte must refuse the tokenization"); + }; - assert!( - prime_memory_with(fixture, &mut context, "Hello\0world") - .is_err_and(|error| error.downcast_ref::().is_some()), - "an interior null byte must surface a tokenization error" + assert_eq!( + Some(failure.downcast::()?), + CString::new("Hello\0world") + .err() + .map(StringToTokenError::NulError) ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 256, + n_ubatch = 64, +)] +fn priming_memory_surfaces_a_batch_add_error(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + assert_eq!( prime_memory_with(fixture, &mut context, &"word ".repeat(300)) .err() @@ -30,16 +54,29 @@ fn prime_memory_surfaces_each_underlying_error(fixture: &LlamaFixture<'_>) -> Re "more tokens than one decode takes must surface an add-sequence error" ); - let filler = "word ".repeat(40); - let mut decode_result = prime_memory_with(fixture, &mut context, &filler); - let mut attempts = 0; - while decode_result.is_ok() && attempts < 16 { - decode_result = prime_memory_with(fixture, &mut context, &filler); - attempts += 1; - } - assert!( - decode_result.is_err(), - "filling the context past its window must surface a decode error" + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 256, + n_ubatch = 64, +)] +fn priming_memory_surfaces_a_decode_error(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_memory(fixture, &mut context)?; + + let Err(failure) = prime_memory(fixture, &mut context) else { + bail!("decoding the primed sequence again from its first position must fail"); + }; + + assert_eq!( + failure.downcast::()?, + DecodeError::BatchInvalid ); Ok(()) diff --git a/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs b/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs index ef3ab6a1d..0c9429286 100644 --- a/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs +++ b/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs @@ -2,6 +2,7 @@ use anyhow::Result; use llama_cpp_bindings::error::memory_seq_rm_error::MemorySeqRmError; use llama_cpp_bindings_tests::prime_memory::prime_memory; +use llama_cpp_bindings_tests::two_sequence_context::two_sequence_context; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -80,11 +81,15 @@ fn memory_seq_rm_accepts_an_explicit_range(fixture: &LlamaFixture<'_>) -> Result n_ubatch = 128, )] fn memory_seq_cp_copies_a_whole_primed_sequence(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; + let mut context = two_sequence_context(fixture)?; prime_memory(fixture, &mut context)?; assert_eq!(context.memory_seq_cp(0, 1, None, None), Ok(())); + assert_eq!( + context.memory_seq_pos_max(1)?, + context.memory_seq_pos_max(0)? + ); Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/model_loading_errors.rs b/llama-cpp-bindings-tests/tests/model_loading_errors.rs index 324e968c0..49178aacd 100644 --- a/llama-cpp-bindings-tests/tests/model_loading_errors.rs +++ b/llama-cpp-bindings-tests/tests/model_loading_errors.rs @@ -8,7 +8,6 @@ use std::path::PathBuf; use anyhow::Result; -use llama_cpp_bindings::error::llama_lora_adapter_init_error::LlamaLoraAdapterInitError; use llama_cpp_bindings::error::llama_model_load_error::LlamaModelLoadError; use llama_cpp_bindings::model::LlamaModel; use llama_cpp_bindings::model::params::LlamaModelParams; @@ -23,15 +22,17 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 128, n_ubatch = 64, )] -fn load_model_with_invalid_path_returns_error(fixture: &LlamaFixture<'_>) { +fn load_model_with_a_missing_file_reports_it_not_found(fixture: &LlamaFixture<'_>) { let model_params = LlamaModelParams::default(); let result = LlamaModel::load_from_file(fixture.backend, "/nonexistent/model.gguf", &model_params); - assert!(matches!( - result.unwrap_err(), - LlamaModelLoadError::FileNotFound(path) if path == Path::new("/nonexistent/model.gguf"), - )); + assert_eq!( + result.err(), + Some(LlamaModelLoadError::FileNotFound(PathBuf::from( + "/nonexistent/model.gguf" + ))) + ); } #[llama_test( @@ -42,7 +43,7 @@ fn load_model_with_invalid_path_returns_error(fixture: &LlamaFixture<'_>) { n_batch = 128, n_ubatch = 64, )] -fn load_model_with_invalid_file_content_returns_unloadable_or_reported( +fn load_model_with_invalid_file_content_reports_it_unloadable( fixture: &LlamaFixture<'_>, ) -> Result<()> { let model_params = LlamaModelParams::default(); @@ -51,11 +52,8 @@ fn load_model_with_invalid_file_content_returns_unloadable_or_reported( let result = LlamaModel::load_from_file(fixture.backend, &dummy_path, &model_params); - assert!(matches!( - result.unwrap_err(), - LlamaModelLoadError::Unloadable | LlamaModelLoadError::Reported { .. }, - )); - let _ = fs::remove_file(&dummy_path); + assert_eq!(result.err(), Some(LlamaModelLoadError::Unloadable)); + fs::remove_file(&dummy_path)?; Ok(()) } @@ -67,72 +65,17 @@ fn load_model_with_invalid_file_content_returns_unloadable_or_reported( n_batch = 128, n_ubatch = 64, )] -fn load_model_with_non_utf8_path_returns_path_to_str_error(fixture: &LlamaFixture<'_>) { +fn load_model_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) { let model_params = LlamaModelParams::default(); let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.gguf")); let result = LlamaModel::load_from_file(fixture.backend, non_utf8_path, &model_params); - assert!(matches!( - result.unwrap_err(), - LlamaModelLoadError::PathToStrError(path) if path == non_utf8_path.to_path_buf() - )); -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn lora_adapter_init_with_invalid_path_returns_error(fixture: &LlamaFixture<'_>) { - let result = fixture - .model - .lora_adapter_init("/nonexistent/path/lora.gguf"); - assert_eq!( - result.unwrap_err(), - LlamaLoraAdapterInitError::FileNotFound(PathBuf::from("/nonexistent/path/lora.gguf")) - ); -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn lora_adapter_init_with_invalid_gguf_returns_unloadable( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let dummy_path = env::temp_dir().join("llama_test_dummy_lora.gguf"); - fs::write(&dummy_path, b"not a valid gguf")?; - - let result = fixture.model.lora_adapter_init(&dummy_path); - - assert_eq!(result.unwrap_err(), LlamaLoraAdapterInitError::Unloadable); - let _ = fs::remove_file(&dummy_path); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn lora_adapter_init_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) { - let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.gguf")); - let result = fixture.model.lora_adapter_init(non_utf8_path); - assert_eq!( - result.unwrap_err(), - LlamaLoraAdapterInitError::PathToStrError(non_utf8_path.to_path_buf()) + result.err(), + Some(LlamaModelLoadError::PathToStrError( + non_utf8_path.to_path_buf() + )) ); } @@ -144,7 +87,7 @@ fn lora_adapter_init_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_> n_batch = 128, n_ubatch = 64, )] -fn load_model_with_a_nul_byte_in_the_path_returns_error(fixture: &LlamaFixture<'_>) { +fn load_model_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) { assert_eq!( LlamaModel::load_from_file( fixture.backend, @@ -157,23 +100,3 @@ fn load_model_with_a_nul_byte_in_the_path_returns_error(fixture: &LlamaFixture<' .map(LlamaModelLoadError::NullError) ); } - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn lora_adapter_init_with_a_nul_byte_in_the_path_returns_error(fixture: &LlamaFixture<'_>) { - assert_eq!( - fixture - .model - .lora_adapter_init("/adapters/adapter\0.gguf") - .err(), - CString::new("/adapters/adapter\0.gguf") - .err() - .map(LlamaLoraAdapterInitError::NullError) - ); -} diff --git a/llama-cpp-bindings-tests/tests/model_metadata.rs b/llama-cpp-bindings-tests/tests/model_metadata.rs index bbddefdfb..8f3ae13f3 100644 --- a/llama-cpp-bindings-tests/tests/model_metadata.rs +++ b/llama-cpp-bindings-tests/tests/model_metadata.rs @@ -1,8 +1,14 @@ +use std::ffi::CString; + use anyhow::Result; +use llama_cpp_bindings::error::meta_val_error::MetaValError; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; +const INITIAL_META_VALUE_BUFFER_BYTES: usize = 256; +const OUT_OF_RANGE_META_INDEX_RETURN: i32 = -1; + #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -147,9 +153,13 @@ fn the_first_metadata_entry_has_a_nonempty_value(fixture: &LlamaFixture<'_>) -> n_batch = 128, n_ubatch = 64, )] -fn meta_key_by_index_out_of_range_returns_error(fixture: &LlamaFixture<'_>) { - let result = fixture.model.meta_key_by_index(999_999); - assert!(result.is_err()); +fn meta_key_by_index_out_of_range_reports_the_negative_llama_cpp_return( + fixture: &LlamaFixture<'_>, +) { + assert_eq!( + fixture.model.meta_key_by_index(999_999), + Err(MetaValError::NegativeReturn(OUT_OF_RANGE_META_INDEX_RETURN)) + ); } #[llama_test( @@ -184,9 +194,13 @@ fn meta_key_by_index_out_of_range_returns_error(fixture: &LlamaFixture<'_>) { n_batch = 128, n_ubatch = 64, )] -fn meta_val_str_by_index_out_of_range_returns_error(fixture: &LlamaFixture<'_>) { - let result = fixture.model.meta_val_str_by_index(999_999); - assert!(result.is_err()); +fn meta_val_str_by_index_out_of_range_reports_the_negative_llama_cpp_return( + fixture: &LlamaFixture<'_>, +) { + assert_eq!( + fixture.model.meta_val_str_by_index(999_999), + Err(MetaValError::NegativeReturn(OUT_OF_RANGE_META_INDEX_RETURN)) + ); } #[llama_test( @@ -261,16 +275,15 @@ fn meta_val_str_returns_value_for_known_key(fixture: &LlamaFixture<'_>) -> Resul n_batch = 128, n_ubatch = 64, )] -fn meta_val_str_with_long_value_triggers_buffer_resize(fixture: &LlamaFixture<'_>) { - let model = fixture.model; - let count = model.meta_count(); +fn meta_val_str_reads_a_value_longer_than_its_initial_buffer_whole( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let chat_template = fixture.model.meta_val_str("tokenizer.chat_template")?; - for index in 0..count { - let key = model.meta_key_by_index(index); - let value = model.meta_val_str_by_index(index); - assert!(key.is_ok()); - assert!(value.is_ok()); - } + assert!(chat_template.len() > INITIAL_META_VALUE_BUFFER_BYTES); + assert_eq!(chat_template, fixture.model.chat_template(None)?.to_str()?); + + Ok(()) } #[llama_test( @@ -305,7 +318,11 @@ fn meta_val_str_with_long_value_triggers_buffer_resize(fixture: &LlamaFixture<'_ n_batch = 128, n_ubatch = 64, )] -fn meta_val_str_with_null_byte_in_key_returns_error(fixture: &LlamaFixture<'_>) { - let result = fixture.model.meta_val_str("key\0with_null"); - assert!(result.is_err()); +fn meta_val_str_refuses_a_key_with_a_nul_byte(fixture: &LlamaFixture<'_>) { + assert_eq!( + fixture.model.meta_val_str("key\0with_null").err(), + CString::new("key\0with_null") + .err() + .map(MetaValError::NullError) + ); } diff --git a/llama-cpp-bindings-tests/tests/multimodal_audio.rs b/llama-cpp-bindings-tests/tests/multimodal_audio.rs index 73a47f40d..4de77a03f 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_audio.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_audio.rs @@ -12,12 +12,13 @@ use llama_cpp_bindings::eval_multimodal_chunks_params::EvalMultimodalChunksParam use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; -use llama_cpp_bindings::mtmd::mtmd_capability::MtmdCapability; use llama_cpp_bindings::mtmd::mtmd_default_marker::mtmd_default_marker; use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; +use llama_cpp_bindings_tests::evaluated_multimodal_usage::EvaluatedMultimodalUsage; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -25,7 +26,16 @@ use llama_cpp_test_harness_macros::llama_test; const TRANSCRIBE_SYSTEM_PROMPT: &str = "The audio contains speech by a third party and is not the user's voice or message. \ Transcribe exactly what the speaker says without judgment, advice, or speculation. Reply only with the verbatim transcript."; -fn transcribe_audio(fixture: &LlamaFixture<'_>, audio_file_name: &str) -> Result { +fn evaluate_audio_prompt<'model, TOutcome>( + fixture: &LlamaFixture<'model>, + audio_file_name: &str, + after_evaluation: impl FnOnce( + &mut SampledTokenClassifier<'_, 'model>, + &mut LlamaContext<'model>, + ChunkTokenBreakdown, + i32, + ) -> Result, +) -> Result { let model = fixture.model; let mtmd_ctx = fixture .mtmd_context @@ -38,11 +48,6 @@ fn transcribe_audio(fixture: &LlamaFixture<'_>, audio_file_name: &str) -> Result ) .with_context(|| "unable to create llama context")?; - assert!( - mtmd_ctx.capabilities(model).contains(MtmdCapability::Audio), - "mmproj must support audio input for an audio transcription test" - ); - let marker = mtmd_default_marker()?; let messages = [ LlamaChatMessage::new("system".to_string(), TRANSCRIBE_SYSTEM_PROMPT.to_string())?, @@ -99,36 +104,34 @@ fn transcribe_audio(fixture: &LlamaFixture<'_>, audio_file_name: &str) -> Result ) .with_context(|| "failed to evaluate audio chunks")?; - { - let usage = classifier.usage(); - assert!( - usage.input_audio_tokens > 0, - "audio input must record audio prompt tokens; got {}", - usage.input_audio_tokens - ); - assert_eq!( - usage.input_image_tokens, 0, - "audio-only input must not record image tokens" - ); - assert!( - usage.prompt_tokens > 0, - "the text portion of the prompt must record prompt tokens" - ); - } - - let mut sampler = LlamaSampler::greedy()?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position: n_past, - max_generated_tokens: 512, - } - .run(); - - Ok(outcome.generated.text().to_lowercase()) + after_evaluation( + &mut classifier, + &mut context, + ChunkTokenBreakdown::from_chunks(&chunks), + n_past, + ) +} + +fn transcribe_audio(fixture: &LlamaFixture<'_>, audio_file_name: &str) -> Result { + evaluate_audio_prompt( + fixture, + audio_file_name, + |classifier, context, _chunk_breakdown, n_past| { + let mut sampler = LlamaSampler::greedy()?; + let mut batch = LlamaBatch::new(ContextLimits::of(context)); + let outcome = ClassifySampleLoop { + classifier, + sampler: &mut sampler, + context, + batch: &mut batch, + initial_position: n_past, + max_generated_tokens: 512, + } + .run(); + + Ok(outcome.generated.text().to_lowercase()) + }, + ) } fn assert_audio_transcription_contains( @@ -182,21 +185,8 @@ fn audio_reaches_the_model_and_produces_a_response(fixture: &LlamaFixture<'_>) - n_ubatch = 512, mmproj_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "mmproj-F16.gguf"), )] -fn audio_mmproj_reports_audio_support(fixture: &LlamaFixture<'_>) { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - assert!( - mtmd_ctx - .capabilities(fixture.model) - .contains(MtmdCapability::Audio), - "an audio mmproj must report audio support" - ); - assert!( - mtmd_ctx.get_audio_sample_rate().is_some(), - "an audio-capable mmproj must report a required sample rate" - ); +fn audio_transcribes_spoken_word(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_audio_transcription_contains(fixture, "quick_brown_fox.wav", "fox") } #[llama_test( @@ -208,10 +198,25 @@ fn audio_mmproj_reports_audio_support(fixture: &LlamaFixture<'_>) { n_ubatch = 512, mmproj_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "mmproj-F16.gguf"), )] -fn audio_transcribes_spoken_word(fixture: &LlamaFixture<'_>) -> Result<()> { - assert_audio_transcription_contains(fixture, "quick_brown_fox.wav", "fox") +fn audio_transcribes_uncommon_sentence(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_audio_transcription_contains(fixture, "orange_cat.wav", "fence") } +#[llama_test( + model_source = HuggingFace( + "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", + "Llama-3.2-1B-Instruct-Q4_K_M.gguf" + ), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace( + "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", + "mmproj-ultravox-v0_5-llama-3_2-1b-f16.gguf" + ), +)] #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -221,6 +226,26 @@ fn audio_transcribes_spoken_word(fixture: &LlamaFixture<'_>) -> Result<()> { n_ubatch = 512, mmproj_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "mmproj-F16.gguf"), )] -fn audio_transcribes_uncommon_sentence(fixture: &LlamaFixture<'_>) -> Result<()> { - assert_audio_transcription_contains(fixture, "orange_cat.wav", "fence") +fn audio_input_records_each_chunk_type_in_its_own_usage_counter( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let EvaluatedMultimodalUsage { + usage, + chunk_breakdown, + } = evaluate_audio_prompt( + fixture, + "quick_brown_fox.wav", + |classifier, _context, chunk_breakdown, _n_past| { + Ok(EvaluatedMultimodalUsage { + usage: *classifier.usage(), + chunk_breakdown, + }) + }, + )?; + + assert_eq!(usage.input_audio_tokens, chunk_breakdown.audio); + assert_eq!(usage.input_image_tokens, chunk_breakdown.image); + assert_eq!(usage.prompt_tokens, chunk_breakdown.text); + + Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs b/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs index adf6ea351..c43d04815 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs @@ -1,12 +1,17 @@ +use std::ffi::CString; use std::fs; use anyhow::Result; use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_bitmap_error::MtmdBitmapError; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; +const LLAMAS_JPG_WIDTH: u32 = 640; +const LLAMAS_JPG_HEIGHT: u32 = 427; + #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -25,7 +30,9 @@ use llama_cpp_test_harness_macros::llama_test; n_ubatch = 64, mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), )] -fn from_buffer_creates_bitmap_from_image_bytes(fixture: &LlamaFixture<'_>) -> Result<()> { +fn from_buffer_decodes_image_bytes_into_an_image_bitmap_of_their_size( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let mtmd_ctx = fixture .mtmd_context .expect("mmproj_file declared in attribute"); @@ -35,8 +42,8 @@ fn from_buffer_creates_bitmap_from_image_bytes(fixture: &LlamaFixture<'_>) -> Re let image_bytes = fs::read(&image_path)?; let bitmap = MtmdBitmap::from_buffer(mtmd_ctx, &image_bytes)?; - assert!(bitmap.nx() > 0); - assert!(bitmap.ny() > 0); + assert_eq!(bitmap.nx(), LLAMAS_JPG_WIDTH); + assert_eq!(bitmap.ny(), LLAMAS_JPG_HEIGHT); assert!(!bitmap.is_audio()); Ok(()) @@ -60,11 +67,15 @@ fn from_buffer_creates_bitmap_from_image_bytes(fixture: &LlamaFixture<'_>) -> Re n_ubatch = 64, mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), )] -fn from_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_>) { +fn from_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) { let mtmd_ctx = fixture .mtmd_context .expect("mmproj_file declared in attribute"); - let result = MtmdBitmap::from_file(mtmd_ctx, "path\0null"); - assert!(result.is_err()); + assert_eq!( + MtmdBitmap::from_file(mtmd_ctx, "path\0null").err(), + CString::new("path\0null") + .err() + .map(MtmdBitmapError::CStringError) + ); } diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs index add4bf500..2c893054e 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs @@ -25,7 +25,7 @@ use llama_cpp_test_harness_macros::llama_test; n_ubatch = 64, mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), )] -fn encode_chunk_succeeds_for_image_chunk(fixture: &LlamaFixture<'_>) -> Result<()> { +fn encode_chunk_encodes_an_image_chunk(fixture: &LlamaFixture<'_>) -> Result<()> { let mtmd_ctx = fixture .mtmd_context .expect("mmproj_file declared in attribute"); @@ -37,16 +37,12 @@ fn encode_chunk_succeeds_for_image_chunk(fixture: &LlamaFixture<'_>) -> Result<( parse_special: true, }; let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + let image_chunk = (0..chunks.len()) + .filter_map(|chunk_index| chunks.get(chunk_index)) + .find(|chunk| chunk.chunk_type() == Ok(MtmdInputChunkType::Image)) + .context("a prompt with an image must tokenize into an image chunk")?; + + assert_eq!(mtmd_ctx.encode_chunk(&image_chunk), Ok(())); - for chunk_index in 0..chunks.len() { - let chunk = chunks - .get(chunk_index) - .context(format!("missing chunk at index {chunk_index}"))?; - if chunk.chunk_type() == Ok(MtmdInputChunkType::Image) { - let result = mtmd_ctx.encode_chunk(&chunk); - assert!(result.is_ok()); - return Ok(()); - } - } Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs index 90421a6bb..05f6d1f38 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs @@ -35,6 +35,54 @@ use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; +struct ImageDimensions { + width: u32, + height: u32, +} + +const EVALUATED_IMAGE_DIMENSIONS: [ImageDimensions; 10] = [ + ImageDimensions { + width: 224, + height: 224, + }, + ImageDimensions { + width: 512, + height: 512, + }, + ImageDimensions { + width: 100, + height: 500, + }, + ImageDimensions { + width: 337, + height: 421, + }, + ImageDimensions { + width: 2, + height: 2, + }, + ImageDimensions { + width: 7, + height: 13, + }, + ImageDimensions { + width: 3, + height: 1000, + }, + ImageDimensions { + width: 1000, + height: 3, + }, + ImageDimensions { + width: 1920, + height: 1080, + }, + ImageDimensions { + width: 4096, + height: 4096, + }, +]; + fn eval_synthetic_bitmap(fixture: &LlamaFixture<'_>, width: u32, height: u32) -> Result<()> { let mtmd_ctx = fixture .mtmd_context @@ -65,7 +113,12 @@ fn eval_synthetic_bitmap(fixture: &LlamaFixture<'_>, width: u32, height: u32) -> (*fixture.context_params).into_llama_context_params(), )?; let n_batch = i32::try_from(llama_ctx.n_batch())?; - chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, false)?; + + assert_eq!( + chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, false)?, + n_positions, + "evaluating a {width}x{height} image must advance past every chunk position" + ); Ok(()) } @@ -521,7 +574,9 @@ fn classifier_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_>) n_ubatch = 512, mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), )] -fn eval_chunks_succeeds_for_a_standard_image(fixture: &LlamaFixture<'_>) -> Result<()> { +fn eval_chunks_advances_past_every_position_of_a_standard_image( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let mtmd_ctx = fixture .mtmd_context .expect("mmproj_file declared in attribute"); @@ -553,9 +608,11 @@ fn eval_chunks_succeeds_for_a_standard_image(fixture: &LlamaFixture<'_>) -> Resu (*fixture.context_params).into_llama_context_params(), )?; let n_batch = i32::try_from(llama_ctx.n_batch())?; - let result = chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, false); - assert!(result.is_ok()); + assert_eq!( + chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, false), + Ok(n_positions) + ); Ok(()) } @@ -578,47 +635,10 @@ fn eval_chunks_succeeds_for_a_standard_image(fixture: &LlamaFixture<'_>) -> Resu n_ubatch = 512, mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), )] -fn eval_chunks_succeeds_for_varied_image_dimensions(fixture: &LlamaFixture<'_>) { - let test_dimensions: [(u32, u32); 4] = [(224, 224), (512, 512), (100, 500), (337, 421)]; - - for (width, height) in test_dimensions { - let result = eval_synthetic_bitmap(fixture, width, height); - assert!( - result.is_ok(), - "dimension {width}x{height} should succeed: {result:?}" - ); - } -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn eval_chunks_supports_extreme_dimensions(fixture: &LlamaFixture<'_>) -> Result<()> { - let extreme_dimensions: [(u32, u32); 6] = [ - (2, 2), - (7, 13), - (3, 1000), - (1000, 3), - (1920, 1080), - (4096, 4096), - ]; - - for (width, height) in extreme_dimensions { +fn eval_chunks_evaluates_images_of_ordinary_and_extreme_dimensions( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + for ImageDimensions { width, height } in EVALUATED_IMAGE_DIMENSIONS { eval_synthetic_bitmap(fixture, width, height) .context(format!("dimension {width}x{height} should succeed"))?; } diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs index 57c8ebcff..e4745ccf7 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs @@ -1,8 +1,12 @@ +use std::ffi::CString; + use anyhow::Context as _; use anyhow::Result; use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_input_chunk::MtmdInputChunk; use llama_cpp_bindings::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::mtmd_input_chunks::MtmdInputChunks; use llama_cpp_bindings::mtmd::mtmd_input_chunks_error::MtmdInputChunksError; use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; use llama_cpp_bindings::mtmd::mtmd_tokenize_error::MtmdTokenizeError; @@ -13,6 +17,13 @@ use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; +fn first_image_chunk(chunks: &MtmdInputChunks) -> Result { + (0..chunks.len()) + .filter_map(|chunk_index| chunks.get(chunk_index)) + .find(|chunk| chunk.chunk_type() == Ok(MtmdInputChunkType::Image)) + .context("a prompt with an image must tokenize into an image chunk") +} + #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -162,18 +173,14 @@ fn text_chunk_id_returns_none(fixture: &LlamaFixture<'_>) -> Result<()> { n_ubatch = 64, mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), )] -fn image_chunk_returns_none_for_text_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { +fn an_image_chunk_has_no_text_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { let chunks = tokenize_prompt_with_synthetic_image(fixture, "Hello <__media__>"); - for chunk_index in 0..chunks.len() { - let chunk = chunks - .get(chunk_index) - .context(format!("missing chunk at index {chunk_index}"))?; - if chunk.chunk_type()? == MtmdInputChunkType::Image { - assert_eq!(chunk.text_tokens_of(MtmdInputChunkType::Image), None); - return Ok(()); - } - } + assert_eq!( + first_image_chunk(&chunks)?.text_tokens_of(MtmdInputChunkType::Image), + None + ); + Ok(()) } @@ -195,17 +202,13 @@ fn image_chunk_returns_none_for_text_tokens(fixture: &LlamaFixture<'_>) -> Resul n_ubatch = 64, mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), )] -fn image_chunk_id_returns_some(fixture: &LlamaFixture<'_>) -> Result<()> { +fn an_image_chunk_carries_the_empty_id_of_a_bitmap_without_one( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let chunks = tokenize_prompt_with_synthetic_image(fixture, "Hello <__media__>"); - for chunk_index in 0..chunks.len() { - let chunk = chunks - .get(chunk_index) - .context(format!("missing chunk at index {chunk_index}"))?; - if chunk.chunk_type() == Ok(MtmdInputChunkType::Image) { - assert!(chunk.id().is_some()); - return Ok(()); - } - } + + assert_eq!(first_image_chunk(&chunks)?.id(), Some(String::new())); + Ok(()) } @@ -263,7 +266,9 @@ fn tokenizing_text_with_an_image_produces_chunks(fixture: &LlamaFixture<'_>) -> n_ubatch = 64, mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), )] -fn tokenize_bitmap_count_mismatch_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { +fn tokenizing_more_bitmaps_than_media_markers_reports_a_tokenization_failure( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let mtmd_ctx = fixture .mtmd_context .expect("mmproj_file declared in attribute"); @@ -274,8 +279,11 @@ fn tokenize_bitmap_count_mismatch_returns_error(fixture: &LlamaFixture<'_>) -> R }; let image_data = vec![128u8; 64 * 64 * 3]; let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; - let result = mtmd_ctx.tokenize(input_text, &[&bitmap]); - assert!(result.is_err()); + + assert_eq!( + mtmd_ctx.tokenize(input_text, &[&bitmap]).err(), + Some(MtmdTokenizeError::MediaPreprocessingOrTokenizationFailed) + ); Ok(()) } @@ -297,7 +305,7 @@ fn tokenize_bitmap_count_mismatch_returns_error(fixture: &LlamaFixture<'_>) -> R n_ubatch = 64, mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), )] -fn tokenize_with_null_byte_in_text_returns_error(fixture: &LlamaFixture<'_>) { +fn tokenize_refuses_text_with_a_nul_byte(fixture: &LlamaFixture<'_>) { let mtmd_ctx = fixture .mtmd_context .expect("mmproj_file declared in attribute"); @@ -306,8 +314,13 @@ fn tokenize_with_null_byte_in_text_returns_error(fixture: &LlamaFixture<'_>) { add_special: true, parse_special: true, }; - let result = mtmd_ctx.tokenize(input_text, &[]); - assert!(result.is_err()); + + assert_eq!( + mtmd_ctx.tokenize(input_text, &[]).err(), + CString::new("text\0null") + .err() + .map(MtmdTokenizeError::CStringError) + ); } #[llama_test( diff --git a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs index 2380088fd..31780b222 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs @@ -13,7 +13,6 @@ use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::LlamaModel; use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; -use llama_cpp_bindings::mtmd::mtmd_capability::MtmdCapability; use llama_cpp_bindings::mtmd::mtmd_context::MtmdContext; use llama_cpp_bindings::mtmd::mtmd_default_marker::mtmd_default_marker; use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; @@ -75,14 +74,6 @@ fn an_image_and_audio_prompt_is_answered_with_both_media_counted_as_input( .mtmd_context .expect("mmproj_file declared in attribute"); - assert_eq!( - mtmd_ctx.capabilities(model), - MtmdCapability::Chat - | MtmdCapability::Vision - | MtmdCapability::Audio - | MtmdCapability::TextGeneration - ); - let image_bitmap = load_fixture_bitmap(mtmd_ctx, "llamas.jpg")?; assert!(!image_bitmap.is_audio(), "llamas.jpg must decode as image"); diff --git a/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs b/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs index ddf9f2421..5747c25e8 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs @@ -9,6 +9,8 @@ use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; +const AUDIO_SAMPLE_RATE_HZ: u32 = 16_000; + #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -136,3 +138,66 @@ fn get_audio_sample_rate_is_none_for_vision_only_mmproj(fixture: &LlamaFixture<' "Qwen 3.5 / 3.6 mmproj has no audio; get_audio_sample_rate must return None" ); } + +#[llama_test( + model_source = HuggingFace( + "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", + "Llama-3.2-1B-Instruct-Q4_K_M.gguf" + ), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace( + "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", + "mmproj-ultravox-v0_5-llama-3_2-1b-f16.gguf" + ), +)] +#[llama_test( + model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "mmproj-F16.gguf"), +)] +fn audio_mmproj_reports_audio_support_at_sixteen_kilohertz(fixture: &LlamaFixture<'_>) { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + assert!( + mtmd_ctx + .capabilities(fixture.model) + .contains(MtmdCapability::Audio), + "an audio mmproj must report audio support" + ); + assert_eq!(mtmd_ctx.get_audio_sample_rate(), Some(AUDIO_SAMPLE_RATE_HZ)); +} + +#[llama_test( + model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "mmproj-F16.gguf"), +)] +fn gemma4_mmproj_reports_chat_vision_audio_and_text_generation_capabilities( + fixture: &LlamaFixture<'_>, +) { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + assert_eq!( + mtmd_ctx.capabilities(fixture.model), + MtmdCapability::Chat + | MtmdCapability::Vision + | MtmdCapability::Audio + | MtmdCapability::TextGeneration + ); +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs b/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs index 38fecd77b..26aed74ec 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs @@ -27,7 +27,7 @@ use llama_cpp_test_harness_macros::llama_test; n_ubatch = 64, mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), )] -fn init_from_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_>) { +fn init_from_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) { let mtmd_params = MtmdContextParams::default(); assert_eq!( diff --git a/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs index c7ce0b540..e66dbb498 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs @@ -244,89 +244,6 @@ fn mistral3_classifier_emits_reasoning_for_multimodal_thinking_prompt( n_ubatch = 512, mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), )] -fn qwen35_classifier_emits_reasoning_for_multimodal_thinking_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 200; - - let model = fixture.model; - let backend = fixture.backend; - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let image_path = fixtures_dir().join("llamas.jpg"); - let image_path_str = image_path - .to_str() - .context("image path is not valid UTF-8")?; - let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; - - let session = media_question_chat_session(model, PROMPT_QUESTION)?; - let input_text = MtmdInputText { - text: session.prompt()?, - add_special: true, - parse_special: true, - }; - - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - - let mut classifier = - SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); - let n_past = classifier.eval_multimodal_chunks( - &chunks, - mtmd_ctx, - &mut context, - EvalMultimodalChunksParams { - start_position: 0, - seq_id: 0, - n_batch: 512, - logits_last: true, - }, - )?; - - let mut sampler = LlamaSampler::chain([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position: n_past, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run(); - - let usage = classifier.usage(); - - if outcome.observed_reasoning == 0 { - bail!( - "Qwen 3.5 multimodal + thinking: classifier must emit at least one Reasoning token \ - when the prompt opens a `` block; outcome={outcome:?}" - ); - } - if usage.reasoning_tokens == 0 { - bail!( - "Qwen 3.5 multimodal + thinking: usage.reasoning_tokens must be non-zero; usage={usage:?}" - ); - } - - Ok(()) -} - #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -336,7 +253,7 @@ fn qwen35_classifier_emits_reasoning_for_multimodal_thinking_prompt( n_ubatch = 512, mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), )] -fn qwen36_classifier_emits_reasoning_for_multimodal_thinking_prompt( +fn qwen_classifier_emits_reasoning_for_multimodal_thinking_prompt( fixture: &LlamaFixture<'_>, ) -> Result<()> { const MAX_GENERATED_TOKENS: i32 = 200; @@ -406,12 +323,13 @@ fn qwen36_classifier_emits_reasoning_for_multimodal_thinking_prompt( if outcome.observed_reasoning == 0 { bail!( - "Qwen 3.6 multimodal + thinking: classifier must emit at least one Reasoning token; outcome={outcome:?}" + "Qwen multimodal + thinking: classifier must emit at least one Reasoning token \ + when the prompt opens a `` block; outcome={outcome:?}" ); } if usage.reasoning_tokens == 0 { bail!( - "Qwen 3.6 multimodal + thinking: usage.reasoning_tokens must be non-zero; usage={usage:?}" + "Qwen multimodal + thinking: usage.reasoning_tokens must be non-zero; usage={usage:?}" ); } diff --git a/llama-cpp-bindings-tests/tests/reranking.rs b/llama-cpp-bindings-tests/tests/reranking.rs index bf2552058..e03e8e440 100644 --- a/llama-cpp-bindings-tests/tests/reranking.rs +++ b/llama-cpp-bindings-tests/tests/reranking.rs @@ -1,21 +1,22 @@ -use anyhow::Context; use anyhow::Result; -use anyhow::bail; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; -use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings_tests::normalize::normalize; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; -fn cosine_similarity(vec_a: &[f32], vec_b: &[f32]) -> f32 { - vec_a +const QUERY: &str = "What is machine learning?"; +const RELEVANT_DOCUMENT: &str = "Machine learning is a subset of artificial intelligence."; +const UNRELATED_DOCUMENT: &str = "The weather today is sunny and warm."; + +fn cosine_similarity(left_embedding: &[f32], right_embedding: &[f32]) -> f32 { + normalize(left_embedding) .iter() - .zip(vec_b.iter()) + .zip(normalize(right_embedding).iter()) .map(|(left, right)| left * right) .sum::() } @@ -27,107 +28,40 @@ fn cosine_similarity(vec_a: &[f32], vec_b: &[f32]) -> f32 { n_ctx = 512, n_batch = 2048, n_ubatch = 512, - n_seq_max = 2, + n_seq_max = 3, n_threads_batch = 8, embeddings = true, )] -fn reranking_produces_scores(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - - let query = "What is machine learning?"; - let documents = [ - "Machine learning is a subset of artificial intelligence.", - "The weather today is sunny and warm.", - ]; - - let document_count = documents.len(); - assert_eq!( - u32::try_from(document_count)?, - fixture.context_params.n_seq_max, - "attribute n_seq_max must match the document count this trial expects", - ); - - let mut ctx = LlamaContext::from_model( - model, +fn the_relevant_document_embeds_closer_to_the_query_than_an_unrelated_one( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, fixture.backend, (*fixture.context_params).into_llama_context_params(), - ) - .with_context(|| "unable to create context")?; - - let prompt_lines: Vec = documents - .iter() - .map(|document| format!("{query}{document}")) - .collect(); - - let tokens_lines_list = prompt_lines - .iter() - .map(|line| model.str_to_token(line, AddBos::Always, ParseSpecialTokens::Always)) - .collect::, _>>() - .with_context(|| "failed to tokenize prompts")?; - - let n_ctx = usize::try_from(ctx.n_ctx())?; - - if tokens_lines_list.iter().any(|tokens| n_ctx < tokens.len()) { - bail!("one of the provided prompts exceeds the size of the context window"); - } + )?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - let mut classifier = SampledTokenClassifier::undeterminable(model); - let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); + for (sequence_id, text) in (0..).zip([QUERY, RELEVANT_DOCUMENT, UNRELATED_DOCUMENT]) { + let tokens = + fixture + .model + .str_to_token(text, AddBos::Always, ParseSpecialTokens::Always)?; - for (sequence_index, tokens) in tokens_lines_list.iter().enumerate() { - classifier.feed_prompt_sequence_to_batch( - &mut batch, - tokens, - i32::try_from(sequence_index)?, - false, - )?; + batch.add_sequence(&tokens, sequence_id, false)?; } - let total_tokens: usize = tokens_lines_list.iter().map(Vec::len).sum(); - let total_token_count = u64::try_from(total_tokens)?; + context.decode(&batch)?; - assert_eq!(classifier.pending_prompt_tokens(), total_token_count); - assert_eq!(classifier.usage().prompt_tokens, 0); - - ctx.memory_clear(); - ctx.decode(&batch) - .with_context(|| "llama_decode() failed")?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, total_token_count); - - let mut embeddings = Vec::with_capacity(document_count); - - for sequence_index in 0..document_count { - let raw_embedding = ctx - .embeddings_seq_ith(i32::try_from(sequence_index)?) - .with_context(|| "failed to get sequence embeddings")?; - embeddings.push(normalize(raw_embedding)); - } - - assert_eq!( - embeddings.len(), - document_count, - "should produce one embedding per document" - ); - - for (index, embedding) in embeddings.iter().enumerate() { - assert!( - !embedding.is_empty(), - "embedding {index} should not be empty" - ); - } - - let similarity = cosine_similarity(&embeddings[0], &embeddings[1]); + let query_embedding = context.embeddings_seq_ith(0)?; + let relevant_similarity = cosine_similarity(query_embedding, context.embeddings_seq_ith(1)?); + let unrelated_similarity = cosine_similarity(query_embedding, context.embeddings_seq_ith(2)?); assert!( - similarity.is_finite(), - "cosine similarity should be a finite number" + relevant_similarity > unrelated_similarity, + "the relevant document must rank above the unrelated one; \ + relevant={relevant_similarity} unrelated={unrelated_similarity}" ); - let usage = classifier.into_usage(); - assert_eq!(usage.prompt_tokens, total_token_count); - assert_eq!(usage.completion_tokens(), 0); - Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/session_file_path_validation.rs b/llama-cpp-bindings-tests/tests/session_file_path_validation.rs index 66aa6b10e..2fbcac9b2 100644 --- a/llama-cpp-bindings-tests/tests/session_file_path_validation.rs +++ b/llama-cpp-bindings-tests/tests/session_file_path_validation.rs @@ -1,9 +1,14 @@ +use std::ffi::CString; use std::ffi::OsStr; use std::os::unix::ffi::OsStrExt; use std::path::Path; use anyhow::Result; +use llama_cpp_bindings::context::load_seq_state_error::LoadSeqStateError; +use llama_cpp_bindings::context::load_session_error::LoadSessionError; +use llama_cpp_bindings::context::save_seq_state_error::SaveSeqStateError; +use llama_cpp_bindings::context::save_session_error::SaveSessionError; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -39,13 +44,17 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 512, n_ubatch = 128, )] -fn state_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { +fn state_save_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) -> Result<()> { let context = fixture.build_context()?; let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); - let result = context.state_save_file(non_utf8_path, &[]); - assert!(result.is_err()); + assert_eq!( + context.state_save_file(non_utf8_path, &[]).err(), + Some(SaveSessionError::PathToStrError( + non_utf8_path.to_path_buf() + )) + ); Ok(()) } @@ -82,13 +91,17 @@ fn state_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) n_batch = 512, n_ubatch = 128, )] -fn state_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { +fn state_load_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); - let result = context.state_load_file(non_utf8_path, 512); - assert!(result.is_err()); + assert_eq!( + context.state_load_file(non_utf8_path, 512).err(), + Some(LoadSessionError::PathToStrError( + non_utf8_path.to_path_buf() + )) + ); Ok(()) } @@ -125,13 +138,17 @@ fn state_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) n_batch = 512, n_ubatch = 128, )] -fn state_seq_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { +fn state_seq_save_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) -> Result<()> { let context = fixture.build_context()?; let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); - let result = context.state_seq_save_file(non_utf8_path, 0, &[]); - assert!(result.is_err()); + assert_eq!( + context.state_seq_save_file(non_utf8_path, 0, &[]).err(), + Some(SaveSeqStateError::PathToStrError( + non_utf8_path.to_path_buf() + )) + ); Ok(()) } @@ -168,13 +185,17 @@ fn state_seq_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<' n_batch = 512, n_ubatch = 128, )] -fn state_seq_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { +fn state_seq_load_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); - let result = context.state_seq_load_file(non_utf8_path, 0, 512); - assert!(result.is_err()); + assert_eq!( + context.state_seq_load_file(non_utf8_path, 0, 512).err(), + Some(LoadSeqStateError::PathToStrError( + non_utf8_path.to_path_buf() + )) + ); Ok(()) } @@ -211,13 +232,17 @@ fn state_seq_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<' n_batch = 512, n_ubatch = 128, )] -fn state_save_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { +fn state_save_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) -> Result<()> { let context = fixture.build_context()?; let path_with_null = Path::new("/tmp/foo\0bar.bin"); - let result = context.state_save_file(path_with_null, &[]); - assert!(result.is_err()); + assert_eq!( + context.state_save_file(path_with_null, &[]).err(), + CString::new("/tmp/foo\0bar.bin") + .err() + .map(SaveSessionError::NullError) + ); Ok(()) } @@ -254,13 +279,17 @@ fn state_save_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<' n_batch = 512, n_ubatch = 128, )] -fn state_load_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { +fn state_load_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; let path_with_null = Path::new("/tmp/foo\0bar.bin"); - let result = context.state_load_file(path_with_null, 512); - assert!(result.is_err()); + assert_eq!( + context.state_load_file(path_with_null, 512).err(), + CString::new("/tmp/foo\0bar.bin") + .err() + .map(LoadSessionError::NullError) + ); Ok(()) } @@ -297,15 +326,17 @@ fn state_load_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<' n_batch = 512, n_ubatch = 128, )] -fn state_seq_save_file_with_null_byte_in_path_returns_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { +fn state_seq_save_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) -> Result<()> { let context = fixture.build_context()?; let path_with_null = Path::new("/tmp/foo\0bar.bin"); - let result = context.state_seq_save_file(path_with_null, 0, &[]); - assert!(result.is_err()); + assert_eq!( + context.state_seq_save_file(path_with_null, 0, &[]).err(), + CString::new("/tmp/foo\0bar.bin") + .err() + .map(SaveSeqStateError::NullError) + ); Ok(()) } @@ -342,15 +373,17 @@ fn state_seq_save_file_with_null_byte_in_path_returns_error( n_batch = 512, n_ubatch = 128, )] -fn state_seq_load_file_with_null_byte_in_path_returns_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { +fn state_seq_load_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) -> Result<()> { let mut context = fixture.build_context()?; let path_with_null = Path::new("/tmp/foo\0bar.bin"); - let result = context.state_seq_load_file(path_with_null, 0, 512); - assert!(result.is_err()); + assert_eq!( + context.state_seq_load_file(path_with_null, 0, 512).err(), + CString::new("/tmp/foo\0bar.bin") + .err() + .map(LoadSeqStateError::NullError) + ); Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs b/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs index 259a22a7d..d9814f73b 100644 --- a/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs +++ b/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs @@ -4,6 +4,8 @@ use std::fs; use anyhow::Result; use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::context::load_seq_state_error::LoadSeqStateError; +use llama_cpp_bindings::context::load_session_error::LoadSessionError; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; @@ -42,117 +44,7 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 512, n_ubatch = 128, )] -fn state_load_file_with_zero_max_tokens_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let tokens = - fixture - .model - .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let session_path = env::temp_dir().join("llama_test_session_zero_max.bin"); - context.state_save_file(&session_path, &tokens)?; - - let result = context.state_load_file(&session_path, 0); - - assert!(result.is_err()); - let _ = fs::remove_file(&session_path); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_load_file_with_zero_max_tokens_returns_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = fixture.build_context()?; - - let tokens = - fixture - .model - .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let session_path = env::temp_dir().join("llama_test_seq_state_zero_max.bin"); - context.state_seq_save_file(&session_path, 0, &tokens)?; - - let result = context.state_seq_load_file(&session_path, 0, 0); - - assert!(result.is_err()); - let _ = fs::remove_file(&session_path); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_load_file_with_insufficient_max_tokens_returns_length_error( +fn state_load_file_with_too_small_a_token_capacity_reports_failed_to_load( fixture: &LlamaFixture<'_>, ) -> Result<()> { let mut context = fixture.build_context()?; @@ -169,10 +61,11 @@ fn state_load_file_with_insufficient_max_tokens_returns_length_error( let session_path = env::temp_dir().join("llama_test_session_insuf.bin"); context.state_save_file(&session_path, &tokens)?; - let result = context.state_load_file(&session_path, 1); - - assert!(result.is_err()); - let _ = fs::remove_file(&session_path); + assert_eq!( + context.state_load_file(&session_path, 1).err(), + Some(LoadSessionError::FailedToLoad) + ); + fs::remove_file(&session_path)?; Ok(()) } @@ -209,7 +102,7 @@ fn state_load_file_with_insufficient_max_tokens_returns_length_error( n_batch = 512, n_ubatch = 128, )] -fn state_seq_load_file_with_insufficient_max_tokens_returns_length_error( +fn state_seq_load_file_with_too_small_a_token_capacity_reports_failed_to_load( fixture: &LlamaFixture<'_>, ) -> Result<()> { let mut context = fixture.build_context()?; @@ -226,10 +119,11 @@ fn state_seq_load_file_with_insufficient_max_tokens_returns_length_error( let session_path = env::temp_dir().join("llama_test_seq_state_insuf.bin"); context.state_seq_save_file(&session_path, 0, &tokens)?; - let result = context.state_seq_load_file(&session_path, 0, 1); - - assert!(result.is_err()); - let _ = fs::remove_file(&session_path); + assert_eq!( + context.state_seq_load_file(&session_path, 0, 1).err(), + Some(LoadSeqStateError::FailedToLoad) + ); + fs::remove_file(&session_path)?; Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/session_files.rs b/llama-cpp-bindings-tests/tests/session_files.rs index 7aedd84b6..0ae26b1f5 100644 --- a/llama-cpp-bindings-tests/tests/session_files.rs +++ b/llama-cpp-bindings-tests/tests/session_files.rs @@ -4,6 +4,10 @@ use std::fs; use anyhow::Result; use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::context::load_seq_state_error::LoadSeqStateError; +use llama_cpp_bindings::context::load_session_error::LoadSessionError; +use llama_cpp_bindings::context::save_seq_state_error::SaveSeqStateError; +use llama_cpp_bindings::context::save_session_error::SaveSessionError; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; @@ -160,12 +164,17 @@ fn a_loaded_sequence_state_file_restores_its_tokens_and_bytes( n_batch = 512, n_ubatch = 128, )] -fn state_load_file_with_nonexistent_file_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { +fn state_load_file_reports_a_missing_file_as_failed_to_load( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let mut context = fixture.build_context()?; - let result = context.state_load_file("/nonexistent/session.bin", 512); - - assert!(result.is_err()); + assert_eq!( + context + .state_load_file("/nonexistent/session.bin", 512) + .err(), + Some(LoadSessionError::FailedToLoad) + ); Ok(()) } @@ -202,14 +211,17 @@ fn state_load_file_with_nonexistent_file_returns_error(fixture: &LlamaFixture<'_ n_batch = 512, n_ubatch = 128, )] -fn state_seq_load_file_with_nonexistent_file_returns_error( +fn state_seq_load_file_reports_a_missing_file_as_failed_to_load( fixture: &LlamaFixture<'_>, ) -> Result<()> { let mut context = fixture.build_context()?; - let result = context.state_seq_load_file("/nonexistent/seq_state.bin", 0, 512); - - assert!(result.is_err()); + assert_eq!( + context + .state_seq_load_file("/nonexistent/seq_state.bin", 0, 512) + .err(), + Some(LoadSeqStateError::FailedToLoad) + ); Ok(()) } @@ -251,9 +263,10 @@ fn state_save_file_to_invalid_directory_returns_failed_to_save( ) -> Result<()> { let context = fixture.build_context()?; - let result = context.state_save_file("/nonexistent_dir/session.bin", &[]); - - assert!(result.is_err()); + assert_eq!( + context.state_save_file("/nonexistent_dir/session.bin", &[]), + Err(SaveSessionError::FailedToSave) + ); Ok(()) } @@ -295,9 +308,10 @@ fn state_seq_save_file_to_invalid_directory_returns_failed_to_save( ) -> Result<()> { let context = fixture.build_context()?; - let result = context.state_seq_save_file("/nonexistent_dir/seq_state.bin", 0, &[]); - - assert!(result.is_err()); + assert_eq!( + context.state_seq_save_file("/nonexistent_dir/seq_state.bin", 0, &[]), + Err(SaveSeqStateError::FailedToSave) + ); Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/special_tokens.rs b/llama-cpp-bindings-tests/tests/special_tokens.rs index f96c15a33..e8ed803a4 100644 --- a/llama-cpp-bindings-tests/tests/special_tokens.rs +++ b/llama-cpp-bindings-tests/tests/special_tokens.rs @@ -3,7 +3,6 @@ use anyhow::Result; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; -use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -39,12 +38,11 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 128, n_ubatch = 64, )] -fn eos_token_differs_from_bos_and_ends_generation(fixture: &LlamaFixture<'_>) -> Result<()> { +fn the_eos_token_differs_from_the_bos_token(fixture: &LlamaFixture<'_>) -> Result<()> { let model = fixture.model; let eos = model.token_eos().context("model declares an EOS token")?; assert_ne!(model.token_bos(), Some(eos)); - assert!(model.is_eog_token(&SampledToken::Content(eos))); Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/state_snapshots.rs b/llama-cpp-bindings-tests/tests/state_snapshots.rs index ee960e690..bd08bd8fe 100644 --- a/llama-cpp-bindings-tests/tests/state_snapshots.rs +++ b/llama-cpp-bindings-tests/tests/state_snapshots.rs @@ -142,59 +142,6 @@ fn state_set_data_consumes_exactly_the_bytes_state_get_data_wrote( Ok(()) } -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_get_size_ext_returns_size_for_decoded_sequence( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = fixture.build_context()?; - - let tokens = - fixture - .model - .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let flags = LlamaStateSeqFlags::empty(); - let size = context.state_seq_get_size_ext(0, &flags); - - assert!(size > 0); - - Ok(()) -} - #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -246,6 +193,7 @@ fn state_seq_set_data_ext_consumes_exactly_the_bytes_state_seq_get_data_ext_wrot let bytes_written = unsafe { context.state_seq_get_data_ext(&mut buffer, 0, &flags) }?; assert!(bytes_written > 0); + assert_eq!(bytes_written, size); let bytes_read = unsafe { context.state_seq_set_data_ext(&buffer, 0, &flags) }?; diff --git a/llama-cpp-bindings-tests/tests/token_attributes.rs b/llama-cpp-bindings-tests/tests/token_attributes.rs index db93128ff..62852b5a0 100644 --- a/llama-cpp-bindings-tests/tests/token_attributes.rs +++ b/llama-cpp-bindings-tests/tests/token_attributes.rs @@ -1,6 +1,7 @@ use anyhow::Context; use anyhow::Result; +use llama_cpp_bindings::llama_token_attrs::LlamaTokenAttrs; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -36,14 +37,13 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 128, n_ubatch = 64, )] -fn token_attr_returns_attrs_for_bos(fixture: &LlamaFixture<'_>) -> Result<()> { +fn the_eos_token_carries_only_the_control_attribute(fixture: &LlamaFixture<'_>) -> Result<()> { let model = fixture.model; - let bos = model.token_bos().context("model declares a BOS token")?; - let attrs = model.token_attr(bos)?; - let bit_repr = format!("{:?}", *attrs); - assert!( - !bit_repr.is_empty(), - "token_attr(bos) must produce Debug output" + let eos = model.token_eos().context("model declares an EOS token")?; + + assert_eq!( + model.token_attr(eos)?, + LlamaTokenAttrs::try_from(llama_cpp_bindings_sys::LLAMA_TOKEN_ATTR_CONTROL)? ); Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/token_candidates.rs b/llama-cpp-bindings-tests/tests/token_candidates.rs index 0a900c3a5..c6da5f933 100644 --- a/llama-cpp-bindings-tests/tests/token_candidates.rs +++ b/llama-cpp-bindings-tests/tests/token_candidates.rs @@ -45,35 +45,6 @@ fn a_token_data_array_holds_one_candidate_per_vocabulary_token( Ok(()) } -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn candidates_ith_returns_n_vocab_entries(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let last_index = i32::try_from(tokens.len() - 1)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&batch)?; - - let count = context.candidates_ith(last_index)?.count(); - - assert_eq!(count, usize::try_from(fixture.model.n_vocab())?); - - Ok(()) -} - #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, diff --git a/llama-cpp-bindings-tests/tests/tokenization.rs b/llama-cpp-bindings-tests/tests/tokenization.rs index 36a6ba08f..b637ef4e3 100644 --- a/llama-cpp-bindings-tests/tests/tokenization.rs +++ b/llama-cpp-bindings-tests/tests/tokenization.rs @@ -1,5 +1,3 @@ -use std::fmt::Write; - use anyhow::Result; use llama_cpp_bindings::model::add_bos::AddBos; @@ -41,7 +39,7 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 128, n_ubatch = 64, )] -fn the_first_token_of_tokenized_text_decodes_to_a_nonempty_piece( +fn the_first_token_of_tokenized_text_decodes_to_its_first_word( fixture: &LlamaFixture<'_>, ) -> Result<()> { let model = fixture.model; @@ -51,57 +49,7 @@ fn the_first_token_of_tokenized_text_decodes_to_a_nonempty_piece( let piece = model.token_to_piece(&SampledToken::Content(tokens[0]), &mut decoder, false, None)?; - assert_ne!(piece, ""); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn str_to_token_grows_buffer_when_initial_estimation_too_small( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let many_short_chars = "a b c d e f g h i j k l"; - let tokens = - fixture - .model - .str_to_token(many_short_chars, AddBos::Always, ParseSpecialTokens::Always)?; - - assert!( - tokens.len() > 8, - "expected regrow; got {} tokens", - tokens.len() - ); + assert_eq!(piece, "hello"); Ok(()) } @@ -184,11 +132,13 @@ fn str_to_token_without_bos_yields_no_more_tokens_than_with_bos( n_batch = 128, n_ubatch = 64, )] -fn str_to_token_with_many_tokens_triggers_buffer_resize(fixture: &LlamaFixture<'_>) -> Result<()> { - let many_numbers = (0..2000).fold(String::new(), |mut accumulator, number| { - let _ = write!(accumulator, "{number} "); - accumulator - }); +fn str_to_token_regrows_its_buffer_for_more_tokens_than_it_estimated( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let many_numbers = (0_u32..2000) + .map(|number| number.to_string()) + .collect::>() + .join(" "); let tokens = fixture diff --git a/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs b/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs index e5e91bd72..6466e6506 100644 --- a/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs +++ b/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs @@ -1,3 +1,4 @@ +use anyhow::Context as _; use anyhow::Result; use llama_cpp_bindings::context::LlamaContext; @@ -9,9 +10,24 @@ use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings::token::LlamaToken; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; +fn most_likely_token(context: &LlamaContext<'_>, token_index: i32) -> Result { + Ok(context + .candidates_ith(token_index)? + .reduce(|most_likely, candidate| { + if candidate.logit() > most_likely.logit() { + candidate + } else { + most_likely + } + }) + .context("a decoded row holds a candidate for every vocabulary token")? + .id()) +} + #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -44,7 +60,9 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 512, n_ubatch = 128, )] -fn sample_without_grammar_produces_multiple_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { +fn sampling_without_a_grammar_counts_every_sampled_token_as_completion( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let model = fixture.model; let mut context = LlamaContext::from_model( model, @@ -66,11 +84,14 @@ fn sample_without_grammar_produces_multiple_tokens(fixture: &LlamaFixture<'_>) - let mut classifier = SampledTokenClassifier::undeterminable(model); let mut outcomes = Vec::new(); let mut sampled_count: u64 = 0; + let mut ended = false; for (position, _) in (batch.n_tokens()..).zip(0..5) { let turn = classifier.sample(&mut sampler, &context, -1, &mut outcomes)?; if turn.progress == GenerationProgress::Ended { + ended = true; + break; } @@ -90,11 +111,10 @@ fn sample_without_grammar_produces_multiple_tokens(fixture: &LlamaFixture<'_>) - sampled_count > 0, "Should produce at least one token without grammar" ); - let usage = classifier.into_usage(); - assert!( - usage.completion_tokens() >= sampled_count, - "completion_tokens ({}) must include the {sampled_count} non-EOG samples", - usage.completion_tokens() + assert_eq!( + classifier.into_usage().completion_tokens(), + sampled_count + u64::from(ended), + "completion_tokens must count every sample, the end of generation included" ); Ok(()) @@ -108,7 +128,9 @@ fn sample_without_grammar_produces_multiple_tokens(fixture: &LlamaFixture<'_>) - n_batch = 2048, n_ubatch = 512, )] -fn a_greedy_sampler_selects_a_candidate_of_a_decoded_row(fixture: &LlamaFixture<'_>) -> Result<()> { +fn a_greedy_sampler_selects_the_most_likely_candidate_of_a_decoded_row( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let mut context = LlamaContext::from_model( fixture.model, fixture.backend, @@ -121,11 +143,15 @@ fn a_greedy_sampler_selects_a_candidate_of_a_decoded_row(fixture: &LlamaFixture< batch.add_sequence(&tokens, 0, false)?; context.decode(&batch)?; - let mut data_array = context.token_data_array_ith(batch.n_tokens() - 1)?; + let last_index = batch.n_tokens() - 1; + let mut data_array = context.token_data_array_ith(last_index)?; let sampler = LlamaSampler::greedy()?; sampler.apply(&mut data_array)?; - assert_ne!(data_array.selected_token(), None); + assert_eq!( + data_array.selected_token(), + Some(most_likely_token(&context, last_index)?) + ); Ok(()) } @@ -170,7 +196,9 @@ fn a_greedy_sampler_selects_a_candidate_of_a_decoded_row(fixture: &LlamaFixture< n_batch = 2048, n_ubatch = 512, )] -fn sampling_the_last_decoded_row_yields_a_token(fixture: &LlamaFixture<'_>) -> Result<()> { +fn sampling_the_last_decoded_row_with_a_greedy_chain_yields_its_most_likely_token( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let mut context = LlamaContext::from_model( fixture.model, fixture.backend, @@ -183,9 +211,12 @@ fn sampling_the_last_decoded_row_yields_a_token(fixture: &LlamaFixture<'_>) -> R batch.add_sequence(&tokens, 0, false)?; context.decode(&batch)?; let mut sampler = LlamaSampler::chain([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; - let token = sampler.sample(&context, batch.n_tokens() - 1)?; + let last_index = batch.n_tokens() - 1; - assert!((0..fixture.model.n_vocab()).contains(&token.0)); + assert_eq!( + sampler.sample(&context, last_index)?, + most_likely_token(&context, last_index)? + ); Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs b/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs index f00da1b47..0296eeb20 100644 --- a/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs +++ b/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs @@ -3,54 +3,6 @@ use anyhow::Result; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn the_vocabulary_iterator_yields_non_negative_token_ids(fixture: &LlamaFixture<'_>) { - let model = fixture.model; - let mut count = 0; - - for (token, _piece_result) in model.tokens(false) { - assert!(token.0 >= 0); - count += 1; - - if count >= 100 { - break; - } - } - - assert_eq!(count, 100); -} - #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, diff --git a/llama-cpp-bindings-types/src/parsed_chat_message.rs b/llama-cpp-bindings-types/src/parsed_chat_message.rs index 0e128309e..de2657e04 100644 --- a/llama-cpp-bindings-types/src/parsed_chat_message.rs +++ b/llama-cpp-bindings-types/src/parsed_chat_message.rs @@ -75,20 +75,24 @@ mod tests { } #[test] - fn message_with_all_three_fields_populated_is_not_empty() { - let parsed = ParsedChatMessage::new( - "hello".to_owned(), - "thinking".to_owned(), - vec![ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - "tool".to_owned(), - ToolCallArguments::ValidJson(json!({})), - )], + fn new_places_each_argument_in_its_field() { + let tool_call = ParsedToolCall::new( + ParsedToolCallId::NotProvidedByModel, + "tool".to_owned(), + ToolCallArguments::ValidJson(json!({})), ); - assert!(!parsed.is_empty()); - assert_eq!(parsed.content, "hello"); - assert_eq!(parsed.reasoning_content, "thinking"); - assert_eq!(parsed.tool_calls.len(), 1); + assert_eq!( + ParsedChatMessage::new( + "hello".to_owned(), + "thinking".to_owned(), + vec![tool_call.clone()] + ), + ParsedChatMessage { + content: "hello".to_owned(), + reasoning_content: "thinking".to_owned(), + tool_calls: vec![tool_call], + } + ); } } diff --git a/llama-cpp-bindings-types/src/token_usage.rs b/llama-cpp-bindings-types/src/token_usage.rs index 5bd59a454..ecd580208 100644 --- a/llama-cpp-bindings-types/src/token_usage.rs +++ b/llama-cpp-bindings-types/src/token_usage.rs @@ -159,6 +159,39 @@ mod tests { use super::TokenUsage; use super::TokenUsageError; + const LEFT_USAGE: TokenUsage = TokenUsage { + prompt_tokens: 20, + cached_prompt_tokens: 2, + input_image_tokens: 3, + input_audio_tokens: 4, + content_tokens: 5, + reasoning_tokens: 6, + tool_call_tokens: 7, + undeterminable_tokens: 8, + }; + + const RIGHT_USAGE: TokenUsage = TokenUsage { + prompt_tokens: 200, + cached_prompt_tokens: 20, + input_image_tokens: 30, + input_audio_tokens: 40, + content_tokens: 50, + reasoning_tokens: 60, + tool_call_tokens: 70, + undeterminable_tokens: 80, + }; + + const COMBINED_USAGE: TokenUsage = TokenUsage { + prompt_tokens: 220, + cached_prompt_tokens: 22, + input_image_tokens: 33, + input_audio_tokens: 44, + content_tokens: 55, + reasoning_tokens: 66, + tool_call_tokens: 77, + undeterminable_tokens: 88, + }; + #[test] fn new_starts_with_all_counters_at_zero() { let usage = TokenUsage::new(); @@ -337,48 +370,8 @@ mod tests { } #[test] - fn add_combines_field_by_field() { - let mut left = TokenUsage::new(); - left.record_prompt_tokens(2); - left.record_cached_prompt_tokens(1) - .expect("1 cached <= 2 prompt is valid"); - left.record_content_token(); - left.record_reasoning_token(); - left.record_tool_call_token(); - left.record_undeterminable_token(); - - let mut right = TokenUsage::new(); - right.record_prompt_tokens(5); - right - .record_cached_prompt_tokens(2) - .expect("2 cached <= 5 prompt is valid"); - right.record_content_token(); - right.record_tool_call_token(); - - let combined = left + right; - - assert_eq!(combined.prompt_tokens, 7); - assert_eq!(combined.cached_prompt_tokens, 3); - assert_eq!(combined.content_tokens, 2); - assert_eq!(combined.reasoning_tokens, 1); - assert_eq!(combined.tool_call_tokens, 2); - assert_eq!(combined.undeterminable_tokens, 1); - } - - #[test] - fn add_combines_image_and_audio_fields_too() { - let mut left = TokenUsage::new(); - left.record_input_image_tokens(3); - left.record_input_audio_tokens(7); - - let mut right = TokenUsage::new(); - right.record_input_image_tokens(4); - right.record_input_audio_tokens(1); - - let combined = left + right; - - assert_eq!(combined.input_image_tokens, 7); - assert_eq!(combined.input_audio_tokens, 8); + fn add_combines_every_field() { + assert_eq!(LEFT_USAGE + RIGHT_USAGE, COMBINED_USAGE); } #[test] @@ -400,17 +393,10 @@ mod tests { } #[test] - fn add_ref_combines_field_by_field() { - let mut left = TokenUsage::new(); - left.record_prompt_tokens(2); - - let mut right = TokenUsage::new(); - right.record_prompt_tokens(5); - let right_ref = &right; - - let combined = left + right_ref; + fn add_ref_combines_every_field() { + let right_usage = &RIGHT_USAGE; - assert_eq!(combined.prompt_tokens, 7); + assert_eq!(LEFT_USAGE + right_usage, COMBINED_USAGE); } #[test] diff --git a/llama-cpp-bindings/src/chat_message_parse_outcome.rs b/llama-cpp-bindings/src/chat_message_parse_outcome.rs index 02a35ed09..bfddd1d58 100644 --- a/llama-cpp-bindings/src/chat_message_parse_outcome.rs +++ b/llama-cpp-bindings/src/chat_message_parse_outcome.rs @@ -56,6 +56,14 @@ mod tests { ) } + fn odd_answer() -> RawChatMessage { + RawChatMessage { + text: "Paris.".to_owned(), + completeness: ChatOutputCompleteness::Finished, + ffi_error_message: "the format differs".to_owned(), + } + } + #[test] fn a_recognized_answer_keeps_its_content_and_reasoning_before_the_tool_calls() { assert_eq!( @@ -100,16 +108,8 @@ mod tests { ); } - fn odd_answer() -> RawChatMessage { - RawChatMessage { - text: "Paris.".to_owned(), - completeness: ChatOutputCompleteness::Finished, - ffi_error_message: "the format differs".to_owned(), - } - } - #[test] - fn an_unrecognized_answer_is_its_raw_text() { + fn an_unrecognized_answer_is_its_raw_message() { assert_eq!( ChatMessageParseOutcome::Unrecognized(odd_answer()).into_recognized(), Err(odd_answer()) diff --git a/llama-cpp-bindings/src/chat_session.rs b/llama-cpp-bindings/src/chat_session.rs index e768b1bcf..299f95741 100644 --- a/llama-cpp-bindings/src/chat_session.rs +++ b/llama-cpp-bindings/src/chat_session.rs @@ -787,7 +787,8 @@ mod tests { } #[test] - fn creating_a_session_reports_each_null_argument_as_a_contract_violation() { + fn creating_a_session_reports_each_null_argument_and_a_null_created_session_as_a_contract_violation() + { for (status, detail) in [ ( llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_TEMPLATES_ARG, diff --git a/llama-cpp-bindings/src/chat_spans.rs b/llama-cpp-bindings/src/chat_spans.rs index a44ab4e89..5270e0198 100644 --- a/llama-cpp-bindings/src/chat_spans.rs +++ b/llama-cpp-bindings/src/chat_spans.rs @@ -237,7 +237,7 @@ mod tests { } #[test] - fn an_untagged_partial_tail_after_content_waits_for_what_follows() { + fn an_untagged_tail_of_a_partial_parse_waits_for_what_follows() { let spans = ChatSpans::new( 8, &[span(ChatSpanKind::Content, 0, 5)], @@ -247,17 +247,6 @@ mod tests { assert_eq!(spans.classify_token(&(5..8)), None); } - #[test] - fn an_untagged_partial_tail_after_reasoning_waits_for_what_follows() { - let spans = ChatSpans::new( - 13, - &[span(ChatSpanKind::Reasoning, 0, 12)], - ChatOutputCompleteness::Partial, - ); - - assert_eq!(spans.classify_token(&(12..13)), None); - } - #[test] fn an_untagged_tail_of_a_finished_parse_after_content_is_a_content_marker() { let spans = ChatSpans::new( @@ -273,7 +262,7 @@ mod tests { } #[test] - fn a_token_without_text_takes_the_role_of_its_position() { + fn a_token_without_text_right_after_reasoning_is_a_reasoning_marker() { assert_eq!( reasoning_then_content(ChatOutputCompleteness::Partial).classify_token(&(12..12)), Some(ChatPieceRole::Marker(SampledTokenSection::Reasoning)) diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/known_marker_candidates.rs b/llama-cpp-bindings/src/chat_template_tool_calls/known_marker_candidates.rs index 7aa8aba1b..4a60ca40b 100644 --- a/llama-cpp-bindings/src/chat_template_tool_calls/known_marker_candidates.rs +++ b/llama-cpp-bindings/src/chat_template_tool_calls/known_marker_candidates.rs @@ -18,3 +18,115 @@ pub fn known_marker_candidates() -> Vec { Gemma4CallBlockFormat::markers(), ] } + +#[cfg(test)] +mod tests { + use serde_json::json; + + use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; + use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; + use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; + use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; + + use super::known_marker_candidates; + use crate::chat_template_tool_calls::gemma4_call_block::Gemma4CallBlockFormat; + use crate::chat_template_tool_calls::glm47_key_value_tags::Glm47KeyValueTagsFormat; + use crate::chat_template_tool_calls::mistral3_arrow_args::Mistral3ArrowArgsFormat; + use crate::chat_template_tool_calls::qwen_xml_tags::QwenXmlTagsFormat; + use crate::tool_call_format::tool_call_format_outcome::ToolCallFormatOutcome; + use crate::tool_call_format::try_parse; + + fn first_candidate_parsing(body: &str) -> Option { + known_marker_candidates().into_iter().find(|candidate| { + matches!(try_parse(body, candidate), ToolCallFormatOutcome::Parsed(_)) + }) + } + + fn weather_in_paris() -> ToolCallFormatOutcome { + ToolCallFormatOutcome::Parsed(vec![ParsedToolCall::new( + ParsedToolCallId::NotProvidedByModel, + "get_weather".to_owned(), + ToolCallArguments::ValidJson(json!({"location": "Paris"})), + )]) + } + + #[test] + fn no_known_candidate_parses_plain_content() { + let plain_content = "Sorry, I cannot help with that request."; + + for candidate in known_marker_candidates() { + assert_eq!( + try_parse(plain_content, &candidate), + ToolCallFormatOutcome::NoMatch, + "expected NoMatch for plain content under candidate {candidate:?}" + ); + } + } + + #[test] + fn qwen_xml_input_resolves_to_the_qwen_xml_tags_format() { + let qwen_input = "\n\ + \n\ + \n\ + Paris\n\ + \n\ + \n\ + "; + let resolved = first_candidate_parsing(qwen_input); + + assert_eq!( + resolved, + Some(QwenXmlTagsFormat::markers()), + "duck-type ordering must resolve Qwen XML via the XmlTags shape (most restrictive \ + shape that requires `Paris<|\"|>}"; + let resolved = first_candidate_parsing(gemma_input); + + assert_eq!(resolved, Some(Gemma4CallBlockFormat::markers())); + assert_eq!( + resolved.map(|candidate| try_parse(gemma_input, &candidate)), + Some(weather_in_paris()) + ); + } +} diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index cf38ed0a5..db4dfc7c7 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -888,7 +888,7 @@ mod check_lora_adapters_result_tests { #[test] fn a_zero_set_adapters_return_code_is_success() { - assert!(check_lora_adapters_result(0).is_ok()); + assert_eq!(check_lora_adapters_result(0), Ok(())); } #[test] @@ -914,17 +914,19 @@ mod nextn_hidden_state_probe_status_to_result_tests { #[test] fn nextn_hidden_state_probe_status_to_result_maps_ok_to_whether_the_graph_produces_it() { assert_eq!( - nextn_hidden_state_probe_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_OK, - true, - ptr::null_mut(), - ), - Ok(true) + [true, false].map(|produces_nextn_hidden_state| { + nextn_hidden_state_probe_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_OK, + produces_nextn_hidden_state, + ptr::null_mut(), + ) + }), + [Ok(true), Ok(false)] ); } #[test] - fn nextn_hidden_state_probe_status_to_result_maps_null_ctx_arg() { + fn nextn_hidden_state_probe_status_to_result_reports_a_null_ctx_arg_as_a_contract_violation() { assert_eq!( nextn_hidden_state_probe_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_CTX_ARG, @@ -940,7 +942,8 @@ mod nextn_hidden_state_probe_status_to_result_tests { } #[test] - fn nextn_hidden_state_probe_status_to_result_maps_null_out_produces_nextn_hidden_state_arg() { + fn nextn_hidden_state_probe_status_to_result_reports_a_null_out_produces_nextn_hidden_state_arg_as_a_contract_violation() + { assert_eq!( nextn_hidden_state_probe_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_PRODUCES_NEXTN_HIDDEN_STATE_ARG, @@ -956,7 +959,8 @@ mod nextn_hidden_state_probe_status_to_result_tests { } #[test] - fn nextn_hidden_state_probe_status_to_result_maps_null_out_error_arg() { + fn nextn_hidden_state_probe_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() + { assert_eq!( nextn_hidden_state_probe_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_ERROR_ARG, @@ -972,7 +976,7 @@ mod nextn_hidden_state_probe_status_to_result_tests { } #[test] - fn nextn_hidden_state_probe_status_to_result_maps_llama_cpp_failed_to_reserve_graph() { + fn nextn_hidden_state_probe_status_to_result_reports_a_failed_graph_reservation() { assert_eq!( nextn_hidden_state_probe_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_FAILED_TO_RESERVE_GRAPH, @@ -984,7 +988,7 @@ mod nextn_hidden_state_probe_status_to_result_tests { } #[test] - fn nextn_hidden_state_probe_status_to_result_maps_error_string_allocation_failed() { + fn nextn_hidden_state_probe_status_to_result_reports_not_enough_memory_for_the_error_string() { assert_eq!( nextn_hidden_state_probe_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_ERROR_STRING_ALLOCATION_FAILED, @@ -996,7 +1000,7 @@ mod nextn_hidden_state_probe_status_to_result_tests { } #[test] - fn nextn_hidden_state_probe_status_to_result_maps_llama_cpp_out_of_memory() { + fn nextn_hidden_state_probe_status_to_result_reports_llama_cpp_running_out_of_memory() { assert_eq!( nextn_hidden_state_probe_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_OUT_OF_MEMORY, @@ -1031,7 +1035,7 @@ mod nextn_hidden_state_probe_status_to_result_tests { } #[test] - fn nextn_hidden_state_probe_status_to_result_maps_llama_cpp_threw_cxx_exception_without_a_message() + fn nextn_hidden_state_probe_status_to_result_reports_an_exception_without_a_message_as_a_contract_violation() { assert_eq!( nextn_hidden_state_probe_status_to_result( @@ -1048,7 +1052,7 @@ mod nextn_hidden_state_probe_status_to_result_tests { } #[test] - fn nextn_hidden_state_probe_status_to_result_maps_an_unknown_status() { + fn nextn_hidden_state_probe_status_to_result_preserves_an_unknown_status() { assert_eq!( nextn_hidden_state_probe_status_to_result(255, false, ptr::null_mut()), Err(FfiStatusError { @@ -1160,7 +1164,7 @@ mod new_context_with_model_status_to_result_tests { } #[test] - fn new_context_with_model_status_to_result_maps_null_model_arg() { + fn new_context_with_model_status_to_result_reports_a_null_model_arg_as_a_contract_violation() { let outcome = new_context_with_model_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_NULL_MODEL_ARG, ptr::null_mut(), @@ -1179,7 +1183,8 @@ mod new_context_with_model_status_to_result_tests { } #[test] - fn new_context_with_model_status_to_result_maps_null_out_ctx_arg() { + fn new_context_with_model_status_to_result_reports_a_null_out_ctx_arg_as_a_contract_violation() + { let outcome = new_context_with_model_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_NULL_OUT_CTX_ARG, ptr::null_mut(), @@ -1198,7 +1203,8 @@ mod new_context_with_model_status_to_result_tests { } #[test] - fn new_context_with_model_status_to_result_maps_null_out_error_arg() { + fn new_context_with_model_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() + { let outcome = new_context_with_model_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_NULL_OUT_ERROR_ARG, ptr::null_mut(), @@ -1217,7 +1223,7 @@ mod new_context_with_model_status_to_result_tests { } #[test] - fn new_context_with_model_status_to_result_maps_llama_cpp_out_of_memory() { + fn new_context_with_model_status_to_result_reports_llama_cpp_running_out_of_memory() { let outcome = new_context_with_model_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), @@ -1360,7 +1366,7 @@ mod process_status_to_result_tests { } #[test] - fn every_status_without_a_message_maps_to_its_error() { + fn every_status_without_a_return_code_or_message_maps_to_its_result() { let statuses = [ llama_cpp_bindings_sys::LLAMA_RS_PROCESS_OK, llama_cpp_bindings_sys::LLAMA_RS_PROCESS_LLAMA_CPP_RETURNED_NONZERO_CODE, diff --git a/llama-cpp-bindings/src/context/kv_cache_type.rs b/llama-cpp-bindings/src/context/kv_cache_type.rs index 270469881..e43fdd230 100644 --- a/llama-cpp-bindings/src/context/kv_cache_type.rs +++ b/llama-cpp-bindings/src/context/kv_cache_type.rs @@ -167,7 +167,7 @@ mod tests { } #[test] - fn every_kv_cache_type_converts_to_its_ggml_type_and_back() { + fn every_kv_cache_type_converts_to_the_matching_ggml_type_and_back() { let all_variants = [ KvCacheType::F32, KvCacheType::F16, @@ -206,11 +206,21 @@ mod tests { KvCacheType::Q20, ]; - for variant in all_variants { - let ggml_type: llama_cpp_bindings_sys::ggml_type = variant.into(); - let back = KvCacheType::from(ggml_type); + let ggml_types = all_variants.map(llama_cpp_bindings_sys::ggml_type::from); - assert_eq!(back, variant); - } + assert_eq!( + ggml_types.map(|ggml_type| { + unsafe { CStr::from_ptr(llama_cpp_bindings_sys::ggml_type_name(ggml_type)) } + .to_str() + .unwrap() + }), + [ + "f32", "f16", "q4_0", "q4_1", "q5_0", "q5_1", "q8_0", "q8_1", "q2_K", "q3_K", + "q4_K", "q5_K", "q6_K", "q8_K", "iq2_xxs", "iq2_xs", "iq3_xxs", "iq1_s", "iq4_nl", + "iq3_s", "iq2_s", "iq4_xs", "i8", "i16", "i32", "i64", "f64", "iq1_m", "bf16", + "tq1_0", "tq2_0", "mxfp4", "nvfp4", "q1_0", "q2_0", + ] + ); + assert_eq!(ggml_types.map(KvCacheType::from), all_variants); } } diff --git a/llama-cpp-bindings/src/context/memory.rs b/llama-cpp-bindings/src/context/memory.rs index 2fe2757d2..ff9640180 100644 --- a/llama-cpp-bindings/src/context/memory.rs +++ b/llama-cpp-bindings/src/context/memory.rs @@ -332,12 +332,13 @@ mod memory_seq_add_status_to_result_tests { #[test] fn add_ok_status_maps_to_ok() { - let result = memory_seq_add_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_OK, - ptr::null_mut(), + assert_eq!( + memory_seq_add_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_OK, + ptr::null_mut(), + ), + Ok(()) ); - - assert!(result.is_ok()); } #[test] @@ -374,7 +375,7 @@ mod memory_seq_add_status_to_result_tests { } #[test] - fn add_llama_cpp_exception_status_without_a_message_is_a_contract_error_with_unknown_message() { + fn add_llama_cpp_exception_status_without_a_message_is_a_contract_violation() { assert_eq!( memory_seq_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION, @@ -418,7 +419,7 @@ mod memory_seq_add_status_to_result_tests { } #[test] - fn memory_seq_add_status_to_result_maps_null_ctx_arg() { + fn memory_seq_add_status_to_result_reports_a_null_ctx_arg_as_a_contract_violation() { let outcome = memory_seq_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_CTX_ARG, ptr::null_mut(), @@ -436,7 +437,7 @@ mod memory_seq_add_status_to_result_tests { } #[test] - fn memory_seq_add_status_to_result_maps_null_model() { + fn memory_seq_add_status_to_result_reports_a_null_model_as_a_contract_violation() { let outcome = memory_seq_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_MODEL, ptr::null_mut(), @@ -454,7 +455,7 @@ mod memory_seq_add_status_to_result_tests { } #[test] - fn memory_seq_add_status_to_result_maps_llama_cpp_out_of_memory() { + fn memory_seq_add_status_to_result_reports_llama_cpp_running_out_of_memory() { let outcome = memory_seq_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), @@ -475,12 +476,13 @@ mod memory_seq_div_status_to_result_tests { #[test] fn div_ok_status_maps_to_ok() { - let result = memory_seq_div_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_OK, - ptr::null_mut(), + assert_eq!( + memory_seq_div_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_OK, + ptr::null_mut(), + ), + Ok(()) ); - - assert!(result.is_ok()); } #[test] @@ -517,7 +519,7 @@ mod memory_seq_div_status_to_result_tests { } #[test] - fn div_llama_cpp_exception_status_without_a_message_is_a_contract_error_with_unknown_message() { + fn div_llama_cpp_exception_status_without_a_message_is_a_contract_violation() { assert_eq!( memory_seq_div_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_LLAMA_CPP_THREW_CXX_EXCEPTION, @@ -561,7 +563,7 @@ mod memory_seq_div_status_to_result_tests { } #[test] - fn memory_seq_div_status_to_result_maps_null_ctx_arg() { + fn memory_seq_div_status_to_result_reports_a_null_ctx_arg_as_a_contract_violation() { let outcome = memory_seq_div_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_CTX_ARG, ptr::null_mut(), @@ -579,7 +581,7 @@ mod memory_seq_div_status_to_result_tests { } #[test] - fn memory_seq_div_status_to_result_maps_null_model() { + fn memory_seq_div_status_to_result_reports_a_null_model_as_a_contract_violation() { let outcome = memory_seq_div_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_MODEL, ptr::null_mut(), @@ -597,7 +599,7 @@ mod memory_seq_div_status_to_result_tests { } #[test] - fn memory_seq_div_status_to_result_maps_llama_cpp_out_of_memory() { + fn memory_seq_div_status_to_result_reports_llama_cpp_running_out_of_memory() { let outcome = memory_seq_div_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), @@ -717,7 +719,7 @@ mod memory_seq_pos_max_status_to_result_tests { } #[test] - fn seq_pos_max_llama_cpp_exception_status_without_a_message_is_a_contract_error_error() { + fn seq_pos_max_llama_cpp_exception_status_without_a_message_is_a_contract_violation() { assert_eq!( memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_LLAMA_CPP_THREW_CXX_EXCEPTION, @@ -764,7 +766,7 @@ mod memory_seq_pos_max_status_to_result_tests { } #[test] - fn memory_seq_pos_max_status_to_result_maps_llama_cpp_out_of_memory() { + fn memory_seq_pos_max_status_to_result_reports_llama_cpp_running_out_of_memory() { let outcome = memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_LLAMA_CPP_OUT_OF_MEMORY, 0, diff --git a/llama-cpp-bindings/src/context/params.rs b/llama-cpp-bindings/src/context/params.rs index 4d56c554d..2d8495c4e 100644 --- a/llama-cpp-bindings/src/context/params.rs +++ b/llama-cpp-bindings/src/context/params.rs @@ -348,7 +348,7 @@ mod tests { } #[test] - fn with_n_ctx_none_sets_zero() { + fn with_n_ctx_none_reads_back_as_no_context_size() { let params = super::LlamaContextParams::default().with_n_ctx(None); assert_eq!(params.n_ctx(), None); @@ -384,7 +384,9 @@ mod tests { #[test] fn with_embeddings_disables() { - let params = super::LlamaContextParams::default().with_embeddings(false); + let params = super::LlamaContextParams::default() + .with_embeddings(true) + .with_embeddings(false); assert!(!params.embeddings()); } @@ -398,7 +400,9 @@ mod tests { #[test] fn with_offload_kqv_enables() { - let params = super::LlamaContextParams::default().with_offload_kqv(true); + let params = super::LlamaContextParams::default() + .with_offload_kqv(false) + .with_offload_kqv(true); assert!(params.offload_kqv()); } @@ -412,33 +416,29 @@ mod tests { #[test] fn with_swa_full_enables() { - let params = super::LlamaContextParams::default().with_swa_full(true); + let params = super::LlamaContextParams::default() + .with_swa_full(false) + .with_swa_full(true); assert!(params.swa_full()); } #[test] - fn with_rope_scaling_type_linear() { - let params = - super::LlamaContextParams::default().with_rope_scaling_type(RopeScalingType::Linear); - - assert_eq!(params.rope_scaling_type(), RopeScalingType::Linear); - } + fn with_rope_scaling_type_sets_each_scaling_type() { + let scaling_types = [ + RopeScalingType::Linear, + RopeScalingType::Yarn, + RopeScalingType::None, + ]; - #[test] - fn with_rope_scaling_type_yarn() { - let params = - super::LlamaContextParams::default().with_rope_scaling_type(RopeScalingType::Yarn); - - assert_eq!(params.rope_scaling_type(), RopeScalingType::Yarn); - } - - #[test] - fn with_rope_scaling_type_none() { - let params = - super::LlamaContextParams::default().with_rope_scaling_type(RopeScalingType::None); - - assert_eq!(params.rope_scaling_type(), RopeScalingType::None); + assert_eq!( + scaling_types.map(|scaling_type| { + super::LlamaContextParams::default() + .with_rope_scaling_type(scaling_type) + .rope_scaling_type() + }), + scaling_types + ); } #[test] @@ -470,38 +470,23 @@ mod tests { } #[test] - fn with_pooling_type_mean() { - let params = super::LlamaContextParams::default().with_pooling_type(LlamaPoolingType::Mean); - - assert_eq!(params.pooling_type(), LlamaPoolingType::Mean); - } - - #[test] - fn with_pooling_type_cls() { - let params = super::LlamaContextParams::default().with_pooling_type(LlamaPoolingType::Cls); - - assert_eq!(params.pooling_type(), LlamaPoolingType::Cls); - } - - #[test] - fn with_pooling_type_last() { - let params = super::LlamaContextParams::default().with_pooling_type(LlamaPoolingType::Last); - - assert_eq!(params.pooling_type(), LlamaPoolingType::Last); - } - - #[test] - fn with_pooling_type_rank() { - let params = super::LlamaContextParams::default().with_pooling_type(LlamaPoolingType::Rank); - - assert_eq!(params.pooling_type(), LlamaPoolingType::Rank); - } - - #[test] - fn with_pooling_type_none() { - let params = super::LlamaContextParams::default().with_pooling_type(LlamaPoolingType::None); + fn with_pooling_type_sets_each_pooling_type() { + let pooling_types = [ + LlamaPoolingType::Mean, + LlamaPoolingType::Cls, + LlamaPoolingType::Last, + LlamaPoolingType::Rank, + LlamaPoolingType::None, + ]; - assert_eq!(params.pooling_type(), LlamaPoolingType::None); + assert_eq!( + pooling_types.map(|pooling_type| { + super::LlamaContextParams::default() + .with_pooling_type(pooling_type) + .pooling_type() + }), + pooling_types + ); } #[test] @@ -526,13 +511,19 @@ mod tests { } #[test] - fn with_flash_attention_policy_sets_value() { - let params = super::LlamaContextParams::default() - .with_flash_attention_policy(llama_cpp_bindings_sys::LLAMA_FLASH_ATTN_TYPE_ENABLED); + fn with_flash_attention_policy_sets_each_policy() { + let policies = [ + llama_cpp_bindings_sys::LLAMA_FLASH_ATTN_TYPE_ENABLED, + llama_cpp_bindings_sys::LLAMA_FLASH_ATTN_TYPE_DISABLED, + ]; assert_eq!( - params.flash_attention_policy(), - llama_cpp_bindings_sys::LLAMA_FLASH_ATTN_TYPE_ENABLED + policies.map(|policy| { + super::LlamaContextParams::default() + .with_flash_attention_policy(policy) + .flash_attention_policy() + }), + policies ); } @@ -563,22 +554,24 @@ mod tests { } #[test] - fn with_cb_eval_sets_callback() { - extern "C" fn test_cb_eval( + fn with_cb_eval_installs_the_given_callback() { + extern "C" fn echo_whether_asked( _tensor: *mut llama_cpp_bindings_sys::ggml_tensor, - _ask: bool, + ask: bool, _user_data: *mut c_void, ) -> bool { - false + ask } - let result = test_cb_eval(ptr::null_mut(), false, ptr::null_mut()); + let params = super::LlamaContextParams::default().with_cb_eval(Some(echo_whether_asked)); - assert!(!result); - - let params = super::LlamaContextParams::default().with_cb_eval(Some(test_cb_eval)); - - assert!(params.context_params.cb_eval.is_some()); + assert_eq!( + params + .context_params + .cb_eval + .map(|cb_eval| unsafe { cb_eval(ptr::null_mut(), true, ptr::null_mut()) }), + Some(true) + ); } #[test] @@ -591,32 +584,19 @@ mod tests { } #[test] - fn with_flash_attention_policy_disabled() { - let params = super::LlamaContextParams::default() - .with_flash_attention_policy(llama_cpp_bindings_sys::LLAMA_FLASH_ATTN_TYPE_DISABLED); + fn with_attention_type_sets_each_attention_type() { + let attention_types = [LlamaAttentionType::Causal, LlamaAttentionType::NonCausal]; assert_eq!( - params.flash_attention_policy(), - llama_cpp_bindings_sys::LLAMA_FLASH_ATTN_TYPE_DISABLED + attention_types.map(|attention_type| { + super::LlamaContextParams::default() + .with_attention_type(attention_type) + .attention_type() + }), + attention_types ); } - #[test] - fn with_attention_type_causal() { - let params = - super::LlamaContextParams::default().with_attention_type(LlamaAttentionType::Causal); - - assert_eq!(params.attention_type(), LlamaAttentionType::Causal); - } - - #[test] - fn with_attention_type_non_causal() { - let params = - super::LlamaContextParams::default().with_attention_type(LlamaAttentionType::NonCausal); - - assert_eq!(params.attention_type(), LlamaAttentionType::NonCausal); - } - #[test] fn with_yarn_ext_factor_sets_value() { let params = super::LlamaContextParams::default().with_yarn_ext_factor(1.5); @@ -654,7 +634,9 @@ mod tests { #[test] fn with_op_offload_enables() { - let params = super::LlamaContextParams::default().with_op_offload(true); + let params = super::LlamaContextParams::default() + .with_op_offload(false) + .with_op_offload(true); assert!(params.op_offload()); } @@ -675,7 +657,9 @@ mod tests { #[test] fn with_kv_unified_disables() { - let params = super::LlamaContextParams::default().with_kv_unified(false); + let params = super::LlamaContextParams::default() + .with_kv_unified(true) + .with_kv_unified(false); assert!(!params.kv_unified()); } diff --git a/llama-cpp-bindings/src/context/session.rs b/llama-cpp-bindings/src/context/session.rs index bf0ff1c73..f0aea554f 100644 --- a/llama-cpp-bindings/src/context/session.rs +++ b/llama-cpp-bindings/src/context/session.rs @@ -410,12 +410,13 @@ mod process_session_load_result_tests { use crate::token::LlamaToken; #[test] - fn session_load_success_within_bounds() { - let tokens = vec![LlamaToken::new(0); 100]; - let result = process_session_load_result(true, 10, 100, tokens); + fn session_load_truncates_the_tokens_to_the_loaded_count() { + let tokens: Vec = (0..100).map(LlamaToken::new).collect(); - assert!(result.is_ok()); - assert_eq!(result.unwrap().len(), 10); + assert_eq!( + process_session_load_result(true, 10, 100, tokens), + Ok((0..10).map(LlamaToken::new).collect()) + ); } #[test] @@ -445,15 +446,20 @@ mod process_session_load_result_tests { mod process_seq_load_result_tests { use super::process_seq_load_result; use crate::context::load_seq_state_error::LoadSeqStateError; + use crate::context::loaded_seq_state::LoadedSeqState; use crate::token::LlamaToken; #[test] - fn seq_load_success_within_bounds() { - let tokens = vec![LlamaToken::new(0); 100]; - let loaded = process_seq_load_result(42, 10, 100, tokens).expect("bounded load succeeds"); + fn seq_load_truncates_the_tokens_to_the_loaded_count_and_reports_the_bytes_read() { + let tokens: Vec = (0..100).map(LlamaToken::new).collect(); - assert_eq!(loaded.tokens.len(), 10); - assert_eq!(loaded.bytes_read, 42); + assert_eq!( + process_seq_load_result(42, 10, 100, tokens), + Ok(LoadedSeqState { + tokens: (0..10).map(LlamaToken::new).collect(), + bytes_read: 42, + }) + ); } #[test] @@ -490,7 +496,7 @@ mod state_data_status_to_result_tests { use crate::context::state_data_error::StateDataError; #[test] - fn state_data_status_to_result_maps_llama_cpp_out_of_memory() { + fn state_data_status_to_result_reports_llama_cpp_running_out_of_memory() { let outcome = state_data_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_LLAMA_CPP_OUT_OF_MEMORY, 0, @@ -532,7 +538,7 @@ mod state_data_status_to_result_tests { } #[test] - fn state_data_status_to_result_maps_error_string_allocation_failed() { + fn state_data_status_to_result_reports_not_enough_memory_for_the_error_string() { assert_eq!( state_data_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_ERROR_STRING_ALLOCATION_FAILED, diff --git a/llama-cpp-bindings/src/error/new_llama_chat_message_error.rs b/llama-cpp-bindings/src/error/new_llama_chat_message_error.rs index d38337bfe..008e7c434 100644 --- a/llama-cpp-bindings/src/error/new_llama_chat_message_error.rs +++ b/llama-cpp-bindings/src/error/new_llama_chat_message_error.rs @@ -1,6 +1,6 @@ use std::ffi::NulError; -#[derive(Debug, thiserror::Error)] +#[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum NewLlamaChatMessageError { #[error("{0}")] NulError(#[from] NulError), diff --git a/llama-cpp-bindings/src/gguf_context.rs b/llama-cpp-bindings/src/gguf_context.rs index e482a7fd8..765060594 100644 --- a/llama-cpp-bindings/src/gguf_context.rs +++ b/llama-cpp-bindings/src/gguf_context.rs @@ -641,11 +641,16 @@ mod tests { .collect() } - #[test] - fn from_file_opens_valid_gguf() { - let context = GgufContext::from_file(fixture_path()); - - assert!(context.is_ok()); + fn bias_and_weight_tensors(test_name: &str) -> SyntheticGgufFile { + SyntheticGgufBuilder::default() + .tensor(b"synthetic.bias", &[2], GGML_TYPE_F16, vec![0; 4]) + .tensor( + b"synthetic.weight", + &[3, 2], + llama_cpp_bindings_sys::GGML_TYPE_F32, + f32_bytes(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]), + ) + .write(test_name) } #[test] @@ -656,19 +661,10 @@ mod tests { } #[test] - fn n_kv_returns_positive_count() { + fn n_kv_counts_every_key_of_the_file() { let context = GgufContext::from_file(fixture_path()).unwrap(); - assert!(context.n_kv() > 0); - } - - #[test] - fn find_key_finds_a_known_key_at_a_non_negative_index() { - let context = GgufContext::from_file(fixture_path()).unwrap(); - let index = context.find_key("general.architecture"); - - assert!(index.is_ok()); - assert!(index.unwrap() >= 0); + assert_eq!(context.n_kv(), 20); } #[test] @@ -713,12 +709,12 @@ mod tests { } #[test] - fn val_str_returns_architecture_value() { + fn val_str_reads_the_architecture_stored_in_the_file() { let context = GgufContext::from_file(fixture_path()).unwrap(); let index = context.find_key("general.architecture").unwrap(); let value = context.val_str(index).unwrap(); - assert_ne!(value, ""); + assert_eq!(value, "bert"); } #[test] @@ -826,22 +822,21 @@ mod tests { assert_eq!(discriminant(&err), utf8_error_disc()); } + #[test] + fn n_tensors_counts_every_tensor_of_the_file() { + let fixture = bias_and_weight_tensors("n_tensors_counts_every_tensor_of_the_file"); + let context = GgufContext::from_file(&fixture.path).unwrap(); + + assert_eq!(context.n_tensors(), 2); + } + #[test] fn read_tensor_f32_returns_shape_and_values() { - let fixture = SyntheticGgufBuilder::default() - .tensor(b"synthetic.bias", &[2], GGML_TYPE_F16, vec![0; 4]) - .tensor( - b"synthetic.weight", - &[3, 2], - llama_cpp_bindings_sys::GGML_TYPE_F32, - f32_bytes(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]), - ) - .write("read_tensor_f32_returns_shape_and_values"); + let fixture = bias_and_weight_tensors("read_tensor_f32_returns_shape_and_values"); let context = GgufContext::from_file(&fixture.path).unwrap(); let tensor = context.read_tensor_f32("synthetic.weight").unwrap(); - assert_eq!(context.n_tensors(), 2); assert_eq!(tensor.shape, [3, 2, 1, 1]); assert_eq!(tensor.values, vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]); } diff --git a/llama-cpp-bindings/src/grammar_matcher.rs b/llama-cpp-bindings/src/grammar_matcher.rs index c15b0037b..32c32495f 100644 --- a/llama-cpp-bindings/src/grammar_matcher.rs +++ b/llama-cpp-bindings/src/grammar_matcher.rs @@ -141,7 +141,6 @@ mod tests { use toktrie::ApproximateTokEnv; use super::GrammarMatcher; - use super::catch_step_panic; use super::panic_payload_message; use super::stop_reason_to_result; use crate::error::grammar_runtime_error::GrammarRuntimeError; @@ -167,7 +166,7 @@ mod tests { } #[test] - fn a_panicking_parser_step_is_reported_with_its_operation() { + fn a_panicking_parser_step_is_reported_with_its_operation_and_message() { let mut matcher = GrammarMatcher::new(letters_a_then_b_parser()); assert_eq!( @@ -207,17 +206,6 @@ mod tests { ); } - #[test] - fn a_panicking_step_is_reported_with_its_operation_and_message() { - assert_eq!( - catch_step_panic("consume_token", || -> u32 { panic!("parser exploded") }), - Err(GrammarRuntimeError::Panicked { - operation: "consume_token", - message: "parser exploded".to_owned(), - }) - ); - } - #[test] fn benign_stop_reasons_are_ok() { for reason in [ @@ -226,7 +214,7 @@ mod tests { StopReason::NoExtensionBias, StopReason::EndOfSentence, ] { - assert!(stop_reason_to_result(reason, String::new()).is_ok()); + assert_eq!(stop_reason_to_result(reason, String::new()), Ok(())); } } diff --git a/llama-cpp-bindings/src/json_schema_to_grammar.rs b/llama-cpp-bindings/src/json_schema_to_grammar.rs index 334c08118..10ad02ff1 100644 --- a/llama-cpp-bindings/src/json_schema_to_grammar.rs +++ b/llama-cpp-bindings/src/json_schema_to_grammar.rs @@ -119,65 +119,72 @@ pub fn json_schema_to_grammar(schema_json: &str) -> Result 0, - "passing !0 must produce at least one unknown bit" + fn unknown_attribute_bits_are_reported() { + assert_eq!( + LlamaTokenAttrs::try_from(!0), + Err(LlamaTokenAttrsFromIntError::UnknownValue(!0x3FF)) ); } diff --git a/llama-cpp-bindings/src/load_backends.rs b/llama-cpp-bindings/src/load_backends.rs index 54634f1a3..0398c287e 100644 --- a/llama-cpp-bindings/src/load_backends.rs +++ b/llama-cpp-bindings/src/load_backends.rs @@ -23,8 +23,6 @@ mod tests { #[test] fn load_backends_does_not_error_with_default_backends_dir() { - let result = load_backends(); - - assert!(result.is_ok()); + assert_eq!(load_backends(), Ok(())); } } diff --git a/llama-cpp-bindings/src/load_backends_error.rs b/llama-cpp-bindings/src/load_backends_error.rs index 3f7c5e6a3..258f52c35 100644 --- a/llama-cpp-bindings/src/load_backends_error.rs +++ b/llama-cpp-bindings/src/load_backends_error.rs @@ -1,7 +1,7 @@ use std::ffi::NulError; use std::path::PathBuf; -#[derive(Debug, thiserror::Error)] +#[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum LoadBackendsError { #[error("backend directory path is not valid UTF-8: {0}")] PathNotUtf8(PathBuf), diff --git a/llama-cpp-bindings/src/load_backends_from_path.rs b/llama-cpp-bindings/src/load_backends_from_path.rs index 6acdde9c2..fbb287667 100644 --- a/llama-cpp-bindings/src/load_backends_from_path.rs +++ b/llama-cpp-bindings/src/load_backends_from_path.rs @@ -24,7 +24,6 @@ pub fn load_backends_from_path(path: &Path) -> Result<(), LoadBackendsError> { mod tests { use std::ffi::CString; use std::ffi::OsStr; - use std::mem::discriminant; use std::os::unix::ffi::OsStrExt; use std::path::PathBuf; @@ -34,19 +33,22 @@ mod tests { #[test] fn load_backends_from_path_returns_path_null_byte_for_embedded_null() { let path = PathBuf::from(OsStr::from_bytes(b"/tmp/foo\0bar")); - let err = load_backends_from_path(&path).unwrap_err(); - let representative = - LoadBackendsError::PathNullByte(CString::new(b"a\0b".to_vec()).unwrap_err()); - assert_eq!(discriminant(&err), discriminant(&representative)); + assert_eq!( + load_backends_from_path(&path), + Err(LoadBackendsError::PathNullByte( + CString::new("/tmp/foo\0bar").unwrap_err() + )) + ); } #[test] fn load_backends_from_path_returns_path_not_utf8_for_invalid_utf8() { let path = PathBuf::from(OsStr::from_bytes(b"/tmp/\xff\xfe")); - let err = load_backends_from_path(&path).unwrap_err(); - let representative = LoadBackendsError::PathNotUtf8(PathBuf::new()); - assert_eq!(discriminant(&err), discriminant(&representative)); + assert_eq!( + load_backends_from_path(&path), + Err(LoadBackendsError::PathNotUtf8(path.clone())) + ); } } diff --git a/llama-cpp-bindings/src/log_options.rs b/llama-cpp-bindings/src/log_options.rs index effa910ab..f46b0f2fb 100644 --- a/llama-cpp-bindings/src/log_options.rs +++ b/llama-cpp-bindings/src/log_options.rs @@ -25,7 +25,7 @@ mod tests { use crate::log_options::LogOptions; #[test] - fn builder_chain_sets_both_flags() { + fn the_builder_disables_logs_and_demotes_info_to_debug() { let options = LogOptions::default() .with_logs_enabled(false) .with_demote_info_to_debug(true); diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index c1f838832..a02528c9c 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -927,7 +927,7 @@ mod cstring_with_validated_len_tests { use crate::error::string_to_token_error::StringToTokenError; #[test] - fn cstring_with_validated_len_null_byte_returns_error() { + fn cstring_with_validated_len_reports_a_nul_byte_in_the_text() { assert_eq!( cstring_with_validated_len("null\0byte").err(), CString::new("null\0byte") @@ -949,29 +949,10 @@ mod cstring_with_validated_len_tests { } } -#[cfg(test)] -mod validate_string_length_for_tokenizer_tests { - use super::validate_string_length_for_tokenizer; - use crate::error::string_to_token_error::StringToTokenError; - - #[test] - fn validate_string_length_overflow_returns_error() { - assert_eq!( - validate_string_length_for_tokenizer(usize::MAX), - i32::try_from(usize::MAX).map_err(StringToTokenError::CIntConversionError) - ); - } -} - #[cfg(test)] mod checked_token_buffer_capacity_tests { use super::checked_token_buffer_capacity; - #[test] - fn checked_token_buffer_capacity_overflow_returns_error() { - assert!(checked_token_buffer_capacity(usize::MAX).is_err()); - } - #[test] fn checked_token_buffer_capacity_in_range_returns_value() { assert_eq!(checked_token_buffer_capacity(8), Ok(8)); @@ -982,11 +963,6 @@ mod checked_token_buffer_capacity_tests { mod checked_token_count_tests { use super::checked_token_count; - #[test] - fn checked_token_count_negative_returns_error() { - assert!(checked_token_count(-1).is_err()); - } - #[test] fn checked_token_count_non_negative_returns_value() { assert_eq!(checked_token_count(5), Ok(5)); @@ -1093,7 +1069,7 @@ mod extract_meta_string_tests { use crate::error::meta_val_error::MetaValError; #[test] - fn returns_error_when_null_terminator_missing() { + fn a_value_without_a_nul_terminator_is_reported_as_a_negative_return() { let result = extract_meta_string( |buf_ptr, buf_len| { let buffer = unsafe { slice::from_raw_parts_mut(buf_ptr.cast::(), buf_len) }; @@ -1109,14 +1085,14 @@ mod extract_meta_string_tests { } #[test] - fn returns_error_for_negative_return_value() { + fn a_negative_return_value_is_reported_as_is() { let result = extract_meta_string(|_buf_ptr, _buf_len| -5, 4); assert_eq!(result.unwrap_err(), MetaValError::NegativeReturn(-5)); } #[test] - fn returns_error_for_invalid_utf8_data() { + fn invalid_utf8_metadata_is_reported_as_a_utf8_error() { let result = extract_meta_string( |buf_ptr, buf_len| { let buffer = unsafe { slice::from_raw_parts_mut(buf_ptr.cast::(), buf_len) }; @@ -1134,7 +1110,7 @@ mod extract_meta_string_tests { } #[test] - fn triggers_buffer_resize_when_returned_len_exceeds_capacity() { + fn a_value_longer_than_the_buffer_is_reread_into_a_larger_buffer() { let initial_capacity: usize = 4; let length_exceeding_initial_capacity = 10; let written_length = 2; @@ -1157,7 +1133,7 @@ mod extract_meta_string_tests { initial_capacity, ); - assert_eq!(result.unwrap(), "hi"); + assert_eq!(result, Ok("hi".to_owned())); } } @@ -1315,7 +1291,7 @@ mod load_model_from_file_status_to_result_tests { } #[test] - fn load_model_from_file_status_to_result_maps_null_path_arg() { + fn load_model_from_file_status_to_result_reports_a_null_path_arg_as_a_contract_violation() { let outcome = unsafe { load_model_from_file_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_NULL_PATH_ARG, @@ -1337,7 +1313,8 @@ mod load_model_from_file_status_to_result_tests { } #[test] - fn load_model_from_file_status_to_result_maps_null_out_model_arg() { + fn load_model_from_file_status_to_result_reports_a_null_out_model_arg_as_a_contract_violation() + { let outcome = unsafe { load_model_from_file_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_NULL_OUT_MODEL_ARG, @@ -1359,7 +1336,8 @@ mod load_model_from_file_status_to_result_tests { } #[test] - fn load_model_from_file_status_to_result_maps_null_out_error_arg() { + fn load_model_from_file_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() + { let outcome = unsafe { load_model_from_file_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_NULL_OUT_ERROR_ARG, @@ -1381,7 +1359,7 @@ mod load_model_from_file_status_to_result_tests { } #[test] - fn load_model_from_file_status_to_result_maps_llama_cpp_out_of_memory() { + fn load_model_from_file_status_to_result_reports_llama_cpp_running_out_of_memory() { let outcome = unsafe { load_model_from_file_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_LLAMA_CPP_OUT_OF_MEMORY, @@ -1487,7 +1465,7 @@ mod tokenize_status_to_result_tests { } #[test] - fn tokenize_status_to_result_maps_null_vocab_arg() { + fn tokenize_status_to_result_reports_a_null_vocab_arg_as_a_contract_violation() { let outcome = unsafe { tokenize_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_VOCAB_ARG, @@ -1508,7 +1486,7 @@ mod tokenize_status_to_result_tests { } #[test] - fn tokenize_status_to_result_maps_null_text_arg() { + fn tokenize_status_to_result_reports_a_null_text_arg_as_a_contract_violation() { let outcome = unsafe { tokenize_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_TEXT_ARG, @@ -1529,7 +1507,7 @@ mod tokenize_status_to_result_tests { } #[test] - fn tokenize_status_to_result_maps_null_out_returned_count_arg() { + fn tokenize_status_to_result_reports_a_null_out_returned_count_arg_as_a_contract_violation() { let outcome = unsafe { tokenize_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_OUT_RETURNED_COUNT_ARG, @@ -1550,7 +1528,7 @@ mod tokenize_status_to_result_tests { } #[test] - fn tokenize_status_to_result_maps_null_out_error_arg() { + fn tokenize_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() { let outcome = unsafe { tokenize_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_OUT_ERROR_ARG, @@ -1571,7 +1549,7 @@ mod tokenize_status_to_result_tests { } #[test] - fn tokenize_status_to_result_maps_llama_cpp_out_of_memory() { + fn tokenize_status_to_result_reports_llama_cpp_running_out_of_memory() { let outcome = unsafe { tokenize_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_LLAMA_CPP_OUT_OF_MEMORY, diff --git a/llama-cpp-bindings/src/model/llama_chat_message.rs b/llama-cpp-bindings/src/model/llama_chat_message.rs index 3844b34fc..f692f4eca 100644 --- a/llama-cpp-bindings/src/model/llama_chat_message.rs +++ b/llama-cpp-bindings/src/model/llama_chat_message.rs @@ -21,33 +21,39 @@ impl LlamaChatMessage { #[cfg(test)] mod tests { + use std::ffi::CString; + + use crate::error::new_llama_chat_message_error::NewLlamaChatMessageError; use crate::model::llama_chat_message::LlamaChatMessage; #[test] fn a_role_and_content_without_nul_bytes_form_a_message() { - let message = LlamaChatMessage::new("user".to_string(), "hello".to_string()); - - assert!(message.is_ok()); - } - - #[test] - fn null_byte_in_role_returns_error() { - let message = LlamaChatMessage::new("us\0er".to_string(), "hello".to_string()); - - assert!(message.is_err()); + assert_eq!( + LlamaChatMessage::new("user".to_string(), "hello".to_string()), + Ok(LlamaChatMessage { + role: c"user".to_owned(), + content: c"hello".to_owned(), + }) + ); } #[test] - fn null_byte_in_content_returns_error() { - let message = LlamaChatMessage::new("user".to_string(), "hel\0lo".to_string()); - - assert!(message.is_err()); + fn a_nul_byte_in_the_role_is_reported() { + assert_eq!( + LlamaChatMessage::new("us\0er".to_string(), "hello".to_string()), + Err(NewLlamaChatMessageError::NulError( + CString::new("us\0er").unwrap_err() + )) + ); } #[test] - fn an_empty_role_and_content_form_a_message() { - let message = LlamaChatMessage::new(String::new(), String::new()); - - assert!(message.is_ok()); + fn a_nul_byte_in_the_content_is_reported() { + assert_eq!( + LlamaChatMessage::new("user".to_string(), "hel\0lo".to_string()), + Err(NewLlamaChatMessageError::NulError( + CString::new("hel\0lo").unwrap_err() + )) + ); } } diff --git a/llama-cpp-bindings/src/model/llama_chat_template.rs b/llama-cpp-bindings/src/model/llama_chat_template.rs index 3ef57e392..c06125cfb 100644 --- a/llama-cpp-bindings/src/model/llama_chat_template.rs +++ b/llama-cpp-bindings/src/model/llama_chat_template.rs @@ -41,6 +41,8 @@ impl Debug for LlamaChatTemplate { #[cfg(test)] mod tests { + use std::ffi::CString; + use crate::model::llama_chat_template::LlamaChatTemplate; #[test] @@ -52,10 +54,11 @@ mod tests { } #[test] - fn null_byte_returns_error() { - let template = LlamaChatTemplate::new("null\0byte"); - - assert!(template.is_err()); + fn a_nul_byte_in_the_template_is_reported() { + assert_eq!( + LlamaChatTemplate::new("null\0byte"), + Err(CString::new("null\0byte").unwrap_err()) + ); } #[test] diff --git a/llama-cpp-bindings/src/model/llama_lora_adapter.rs b/llama-cpp-bindings/src/model/llama_lora_adapter.rs index fc3f17b6f..2183446bc 100644 --- a/llama-cpp-bindings/src/model/llama_lora_adapter.rs +++ b/llama-cpp-bindings/src/model/llama_lora_adapter.rs @@ -45,22 +45,12 @@ impl Drop for LlamaLoraAdapter<'_> { #[cfg(test)] mod ownership_tests { use std::marker::PhantomData; - use std::mem::ManuallyDrop; use std::ptr; - use std::ptr::NonNull; use crate::model::llama_lora_adapter::LlamaLoraAdapter; #[test] - fn adapter_preserves_the_owned_native_pointer() { - let pointer = NonNull::dangling(); - let adapter = ManuallyDrop::new(unsafe { LlamaLoraAdapter::new(pointer) }); - - assert_eq!(adapter.as_ptr(), pointer.as_ptr()); - } - - #[test] - fn dropping_an_adapter_releases_its_native_pointer() { + fn dropping_an_adapter_holding_a_null_pointer_is_tolerated() { drop(LlamaLoraAdapter { lora_adapter: ptr::null_mut(), model: PhantomData, diff --git a/llama-cpp-bindings/src/model/params.rs b/llama-cpp-bindings/src/model/params.rs index 2194d15b0..4e1f11392 100644 --- a/llama-cpp-bindings/src/model/params.rs +++ b/llama-cpp-bindings/src/model/params.rs @@ -488,15 +488,12 @@ impl Default for LlamaModelParams { mod tests { use std::ffi::CStr; use std::ffi::CString; - use std::mem::discriminant; + use std::ffi::c_char; use std::pin::pin; use std::ptr; - use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; - use super::LLAMA_CPP_MAX_DEVICES; use super::LlamaModelParams; - use super::fit_params_status_to_result; use crate::context::params::LlamaContextParams; use crate::error::fit_error::FitError; use crate::error::llama_cpp_error::LlamaCppError; @@ -511,20 +508,6 @@ mod tests { use crate::model::params::param_override_value::ParamOverrideValue; use crate::model::split_mode::LlamaSplitMode; - fn contract_error_reported_for( - status: llama_cpp_bindings_sys::llama_rs_fit_params_status, - ) -> Option { - fit_params_status_to_result(status, 0, ptr::null_mut()).err() - } - - fn contract_error_with(detail: &'static str) -> FitError { - FfiContractError { - operation: "llama_rs_fit_params", - detail, - } - .into() - } - #[test] fn device_capacity_is_not_smaller_than_the_upstream_limit() { let upstream_limit = max_devices(); @@ -593,19 +576,12 @@ mod tests { } #[test] - fn with_split_mode_none() { + fn with_split_mode_none_disables_splitting() { let params = LlamaModelParams::default().with_split_mode(LlamaSplitMode::None); assert_eq!(params.split_mode(), Ok(LlamaSplitMode::None)); } - #[test] - fn with_split_mode_row() { - let params = LlamaModelParams::default().with_split_mode(LlamaSplitMode::Row); - - assert_eq!(params.split_mode(), Ok(LlamaSplitMode::Row)); - } - #[test] fn with_vocab_only_enables() { let params = LlamaModelParams::default().with_vocab_only(true); @@ -614,8 +590,10 @@ mod tests { } #[test] - fn with_vocab_only_disables() { - let params = LlamaModelParams::default().with_vocab_only(false); + fn with_vocab_only_false_disables_vocab_only_loading() { + let params = LlamaModelParams::default() + .with_vocab_only(true) + .with_vocab_only(false); assert!(!params.vocab_only()); } @@ -628,8 +606,10 @@ mod tests { } #[test] - fn with_no_alloc_disables() { - let params = LlamaModelParams::default().with_no_alloc(false); + fn with_no_alloc_false_reenables_allocation() { + let params = LlamaModelParams::default() + .with_no_alloc(true) + .with_no_alloc(false); assert!(!params.no_alloc()); } @@ -652,8 +632,10 @@ mod tests { } #[test] - fn with_load_mtp_disables_mtp_loading() { - let params = LlamaModelParams::default().with_load_mtp(false); + fn with_load_mtp_false_disables_mtp_loading() { + let params = LlamaModelParams::default() + .with_load_mtp(true) + .with_load_mtp(false); assert!(!params.load_mtp()); } @@ -665,35 +647,17 @@ mod tests { assert!(!params.no_alloc()); } - fn debug_output_of_default_params_with(lazy_mode: LlamaLazyMode) -> String { - format!( - "LlamaModelParams {{ n_gpu_layers: -1, main_gpu: 0, vocab_only: false, \ - lazy_mode: Ok({lazy_mode:?}), load_mode: Ok(Auto), load_mtp: false, \ - split_mode: Ok(Layer), devices: [0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, \ - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0], \ - kv_overrides: \"vec of kv_overrides\", .. }}" - ) - } - - #[test] - fn debug_format_contains_field_names() { - let params = LlamaModelParams::default(); - let debug_output = format!("{params:?}"); - - assert_eq!( - debug_output, - debug_output_of_default_params_with(LlamaLazyMode::Auto) - ); - } - #[test] - fn debug_format_includes_lazy_mode() { + fn debug_output_lists_every_field_including_the_lazy_mode() { let params = LlamaModelParams::default().with_lazy_mode(LlamaLazyMode::On); - let debug_output = format!("{params:?}"); assert_eq!( - debug_output, - debug_output_of_default_params_with(LlamaLazyMode::On) + format!("{params:?}"), + "LlamaModelParams { n_gpu_layers: -1, main_gpu: 0, vocab_only: false, \ + lazy_mode: Ok(On), load_mode: Ok(Auto), load_mtp: false, \ + split_mode: Ok(Layer), devices: [0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, \ + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0], \ + kv_overrides: \"vec of kv_overrides\", .. }" ); } @@ -715,18 +679,34 @@ mod tests { #[test] fn with_devices_accepts_an_empty_device_list() { - let params = LlamaModelParams::default().with_devices(&[]); - - assert!(params.is_ok()); - assert_eq!(params.unwrap().devices(), Vec::::new()); + assert_eq!( + LlamaModelParams::default() + .with_devices(&[]) + .map(|params| params.devices()), + Ok(Vec::new()) + ); } #[test] fn add_cpu_buft_override_fills_an_empty_override_slot() { let mut params = pin!(LlamaModelParams::default()); - let result = params.as_mut().add_cpu_buft_override(c"test_pattern"); - assert!(result.is_ok()); + assert_eq!( + params.as_mut().add_cpu_buft_override(c"test_pattern"), + Ok(()) + ); + assert_eq!( + unsafe { CStr::from_ptr(params.buft_overrides[0].pattern) }, + c"test_pattern" + ); + assert_eq!(params.buft_overrides[0].buft, unsafe { + llama_cpp_bindings_sys::ggml_backend_cpu_buffer_type() + }); + assert_eq!(params.buft_overrides[1].pattern, ptr::null()); + assert_eq!( + params.params.tensor_buft_overrides, + params.buft_overrides.as_ptr() + ); } #[test] @@ -759,11 +739,17 @@ mod tests { } #[test] - fn add_cpu_moe_override_fills_an_empty_override_slot() { + fn add_cpu_moe_override_keeps_expert_tensors_in_cpu_memory() { let mut params = pin!(LlamaModelParams::default()); - let result = params.as_mut().add_cpu_moe_override(); - assert!(result.is_ok()); + assert_eq!(params.as_mut().add_cpu_moe_override(), Ok(())); + assert_eq!( + unsafe { CStr::from_ptr(params.buft_overrides[0].pattern) }, + c"\\.ffn_(up|down|gate)_(ch|)exps" + ); + assert_eq!(params.buft_overrides[0].buft, unsafe { + llama_cpp_bindings_sys::ggml_backend_cpu_buffer_type() + }); } #[test] @@ -808,8 +794,7 @@ mod tests { let params = LlamaModelParams::default().with_devices(&[0]).unwrap(); - assert_eq!(params.devices().len(), 1); - assert_eq!(params.devices()[0], 0); + assert_eq!(params.devices(), vec![0]); } #[test] @@ -836,11 +821,11 @@ mod tests { .append_kv_override(key, ParamOverrideValue::Int(1)); assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ModelParamsError::InvalidCharacterInKey { - byte: 0, - reason: String::new(), - }), + result, + Err(ModelParamsError::InvalidCharacterInKey { + byte: 0xff, + reason: c_char::try_from(0xff_u8).unwrap_err().to_string(), + }) ); } @@ -852,11 +837,11 @@ mod tests { let result = params.as_mut().add_cpu_buft_override(key); assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ModelParamsError::InvalidCharacterInKey { - byte: 0, - reason: String::new(), - }), + result, + Err(ModelParamsError::InvalidCharacterInKey { + byte: 0xff, + reason: c_char::try_from(0xff_u8).unwrap_err().to_string(), + }) ); } @@ -886,7 +871,7 @@ mod tests { #[test] #[serial_test::serial] - fn fit_params_invalid_model_path_returns_error() { + fn fit_params_reports_a_missing_model_file_as_aborted() { let _backend = LlamaBackend::init(); let mut params = pin!(LlamaModelParams::default()); let mut context_params = LlamaContextParams::default(); @@ -928,94 +913,6 @@ mod tests { }) ); } - - #[test] - fn fit_params_status_to_result_maps_null_path_model_arg() { - assert_eq!( - contract_error_reported_for( - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_PATH_MODEL_ARG - ), - Some(contract_error_with("was given a null path_model argument")) - ); - } - - #[test] - fn fit_params_status_to_result_maps_null_mparams_arg() { - assert_eq!( - contract_error_reported_for( - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_MPARAMS_ARG - ), - Some(contract_error_with("was given a null mparams argument")) - ); - } - - #[test] - fn fit_params_status_to_result_maps_null_cparams_arg() { - assert_eq!( - contract_error_reported_for( - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_CPARAMS_ARG - ), - Some(contract_error_with("was given a null cparams argument")) - ); - } - - #[test] - fn fit_params_status_to_result_maps_null_out_unrecognized_status_code_arg() { - assert_eq!( - contract_error_reported_for( - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_OUT_UNRECOGNIZED_STATUS_CODE_ARG - ), - Some(contract_error_with( - "was given a null out_unrecognized_status_code argument" - )) - ); - } - - #[test] - fn fit_params_status_to_result_maps_null_out_error_arg() { - assert_eq!( - contract_error_reported_for( - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_OUT_ERROR_ARG - ), - Some(contract_error_with("was given a null out_error argument")) - ); - } - - #[test] - fn fit_params_status_to_result_maps_null_extra_path_model_arg() { - assert_eq!( - contract_error_reported_for( - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_PATH_MODEL_ARG - ), - Some(contract_error_with( - "was given an extra model with a null path_model argument" - )) - ); - } - - #[test] - fn fit_params_status_to_result_maps_null_extra_mparams_arg() { - assert_eq!( - contract_error_reported_for( - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_MPARAMS_ARG - ), - Some(contract_error_with( - "was given an extra model with a null mparams argument" - )) - ); - } - - #[test] - fn fit_params_status_to_result_maps_null_extra_cparams_arg() { - assert_eq!( - contract_error_reported_for( - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_CPARAMS_ARG - ), - Some(contract_error_with( - "was given an extra model with a null cparams argument" - )) - ); - } } #[cfg(test)] @@ -1028,6 +925,20 @@ mod fit_params_status_to_result_tests { use super::fit_params_status_to_result; use crate::error::fit_error::FitError; + fn contract_error_reported_for( + status: llama_cpp_bindings_sys::llama_rs_fit_params_status, + ) -> Option { + fit_params_status_to_result(status, 0, ptr::null_mut()).err() + } + + fn contract_error_with(detail: &'static str) -> FitError { + FfiContractError { + operation: "llama_rs_fit_params", + detail, + } + .into() + } + #[test] fn fit_params_status_ok_returns_ok() { let result = fit_params_status_to_result( @@ -1102,7 +1013,7 @@ mod fit_params_status_to_result_tests { } #[test] - fn fit_params_status_cxx_exception_without_a_message_is_a_contract_error_with_unknown_error() { + fn fit_params_status_reports_an_exception_without_a_message_as_a_contract_violation() { let result = fit_params_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_LLAMA_CPP_THREW_CXX_EXCEPTION, 0, @@ -1142,4 +1053,93 @@ mod fit_params_status_to_result_tests { assert_eq!(outcome.err(), Some(FitError::LlamaCppOutOfMemory)); } + + #[test] + fn fit_params_status_to_result_reports_a_null_path_model_arg_as_a_contract_violation() { + assert_eq!( + contract_error_reported_for( + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_PATH_MODEL_ARG + ), + Some(contract_error_with("was given a null path_model argument")) + ); + } + + #[test] + fn fit_params_status_to_result_reports_a_null_mparams_arg_as_a_contract_violation() { + assert_eq!( + contract_error_reported_for( + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_MPARAMS_ARG + ), + Some(contract_error_with("was given a null mparams argument")) + ); + } + + #[test] + fn fit_params_status_to_result_reports_a_null_cparams_arg_as_a_contract_violation() { + assert_eq!( + contract_error_reported_for( + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_CPARAMS_ARG + ), + Some(contract_error_with("was given a null cparams argument")) + ); + } + + #[test] + fn fit_params_status_to_result_reports_a_null_out_unrecognized_status_code_arg_as_a_contract_violation() + { + assert_eq!( + contract_error_reported_for( + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_OUT_UNRECOGNIZED_STATUS_CODE_ARG + ), + Some(contract_error_with( + "was given a null out_unrecognized_status_code argument" + )) + ); + } + + #[test] + fn fit_params_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() { + assert_eq!( + contract_error_reported_for( + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_OUT_ERROR_ARG + ), + Some(contract_error_with("was given a null out_error argument")) + ); + } + + #[test] + fn fit_params_status_to_result_reports_a_null_extra_path_model_arg_as_a_contract_violation() { + assert_eq!( + contract_error_reported_for( + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_PATH_MODEL_ARG + ), + Some(contract_error_with( + "was given an extra model with a null path_model argument" + )) + ); + } + + #[test] + fn fit_params_status_to_result_reports_a_null_extra_mparams_arg_as_a_contract_violation() { + assert_eq!( + contract_error_reported_for( + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_MPARAMS_ARG + ), + Some(contract_error_with( + "was given an extra model with a null mparams argument" + )) + ); + } + + #[test] + fn fit_params_status_to_result_reports_a_null_extra_cparams_arg_as_a_contract_violation() { + assert_eq!( + contract_error_reported_for( + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_CPARAMS_ARG + ), + Some(contract_error_with( + "was given an extra model with a null cparams argument" + )) + ); + } } diff --git a/llama-cpp-bindings/src/model/params/param_override_value.rs b/llama-cpp-bindings/src/model/params/param_override_value.rs index f8c18a290..a6e13b55b 100644 --- a/llama-cpp-bindings/src/model/params/param_override_value.rs +++ b/llama-cpp-bindings/src/model/params/param_override_value.rs @@ -73,6 +73,7 @@ mod tests { use std::mem; use super::ParamOverrideValue; + use crate::model::params::unknown_kv_override_tag::UnknownKvOverrideTag; #[test] fn each_override_value_is_tagged_with_its_llama_cpp_type() { @@ -199,15 +200,16 @@ mod tests { } #[test] - fn unknown_tag_returns_error() { + fn an_unknown_kv_override_tag_is_reported_with_its_value() { let ffi_override = llama_cpp_bindings_sys::llama_model_kv_override { key: [0; 128], tag: 9999, __bindgen_anon_1: unsafe { mem::zeroed() }, }; - let result = ParamOverrideValue::try_from(&ffi_override); - - assert!(result.is_err()); + assert_eq!( + ParamOverrideValue::try_from(&ffi_override), + Err(UnknownKvOverrideTag(9999)) + ); } } diff --git a/llama-cpp-bindings/src/model/split_mode.rs b/llama-cpp-bindings/src/model/split_mode.rs index a8e831eda..bae6020a2 100644 --- a/llama-cpp-bindings/src/model/split_mode.rs +++ b/llama-cpp-bindings/src/model/split_mode.rs @@ -42,13 +42,17 @@ impl From for llama_cpp_bindings_sys::llama_split_mode { #[cfg(test)] mod tests { use super::LlamaSplitMode; + use crate::model::llama_split_mode_parse_error::LlamaSplitModeParseError; #[test] fn try_from_invalid_reports_the_value() { - let result = LlamaSplitMode::try_from(99); - - assert!(result.is_err()); - assert_eq!(result.unwrap_err().value, 99); + assert_eq!( + LlamaSplitMode::try_from(99), + Err(LlamaSplitModeParseError { + value: 99, + context: "unknown split mode value: 99".to_owned(), + }) + ); } #[test] diff --git a/llama-cpp-bindings/src/mtmd/eval_chunk.rs b/llama-cpp-bindings/src/mtmd/eval_chunk.rs index ea3fa49e6..25a686bb2 100644 --- a/llama-cpp-bindings/src/mtmd/eval_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/eval_chunk.rs @@ -189,7 +189,7 @@ mod eval_chunk_single_status_to_result_tests { } #[test] - fn eval_chunk_single_status_cxx_exception_reports_unknown_error_for_null() { + fn eval_chunk_single_status_reports_an_exception_without_a_message_as_a_contract_violation() { let result = eval_chunk_single_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_THREW_CXX_EXCEPTION, 0, @@ -221,7 +221,7 @@ mod eval_chunk_single_status_to_result_tests { } #[test] - fn eval_chunk_single_status_to_result_maps_null_mtmd_ctx_arg() { + fn eval_chunk_single_status_to_result_reports_a_null_mtmd_ctx_arg_as_a_contract_violation() { let outcome = eval_chunk_single_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_MTMD_CTX_ARG, 0, @@ -241,7 +241,7 @@ mod eval_chunk_single_status_to_result_tests { } #[test] - fn eval_chunk_single_status_to_result_maps_null_llama_ctx_arg() { + fn eval_chunk_single_status_to_result_reports_a_null_llama_ctx_arg_as_a_contract_violation() { let outcome = eval_chunk_single_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_LLAMA_CTX_ARG, 0, @@ -261,7 +261,7 @@ mod eval_chunk_single_status_to_result_tests { } #[test] - fn eval_chunk_single_status_to_result_maps_null_chunk_arg() { + fn eval_chunk_single_status_to_result_reports_a_null_chunk_arg_as_a_contract_violation() { let outcome = eval_chunk_single_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_CHUNK_ARG, 0, @@ -281,7 +281,8 @@ mod eval_chunk_single_status_to_result_tests { } #[test] - fn eval_chunk_single_status_to_result_maps_null_out_new_n_past_arg() { + fn eval_chunk_single_status_to_result_reports_a_null_out_new_n_past_arg_as_a_contract_violation() + { let outcome = eval_chunk_single_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_OUT_NEW_N_PAST_ARG, 0, @@ -301,7 +302,7 @@ mod eval_chunk_single_status_to_result_tests { } #[test] - fn eval_chunk_single_status_to_result_maps_llama_cpp_out_of_memory() { + fn eval_chunk_single_status_to_result_reports_llama_cpp_running_out_of_memory() { let outcome = eval_chunk_single_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_OUT_OF_MEMORY, 0, diff --git a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs index 4951e89b7..c30b16aa3 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs @@ -410,6 +410,8 @@ impl Drop for MtmdBitmap { #[cfg(test)] mod tests { + use std::ffi::CString; + use crate::mtmd::mtmd_bitmap::MtmdBitmap; use crate::mtmd::mtmd_bitmap_error::MtmdBitmapError; @@ -424,7 +426,7 @@ mod tests { } #[test] - fn invalid_data_size_returns_error() { + fn image_data_shorter_than_its_dimensions_is_an_invalid_data_size() { let too_short = vec![0u8; 5]; assert_eq!( @@ -470,7 +472,7 @@ mod tests { } #[test] - fn set_id_changes_id() { + fn set_id_replaces_the_bitmap_id() { let image_data = vec![0u8; 12]; let bitmap = MtmdBitmap::from_image_data(2, 2, &image_data).unwrap(); bitmap.set_id("test_image").unwrap(); @@ -498,29 +500,20 @@ mod tests { } #[test] - fn id_returns_some_by_default() { - let image_data = vec![0u8; 12]; - let bitmap = MtmdBitmap::from_image_data(2, 2, &image_data).unwrap(); - - assert!(bitmap.id().is_some()); - } - - #[test] - fn id_returns_custom_value_after_set() { + fn a_new_bitmap_has_an_empty_id() { let image_data = vec![0u8; 12]; let bitmap = MtmdBitmap::from_image_data(2, 2, &image_data).unwrap(); - bitmap.set_id("my_image").unwrap(); - assert_eq!(bitmap.id(), Some("my_image".to_string())); + assert_eq!(bitmap.id(), Some(String::new())); } #[test] - fn set_id_with_null_byte_returns_error() { + fn set_id_reports_a_nul_byte_in_the_id() { let image_data = vec![0u8; 12]; let bitmap = MtmdBitmap::from_image_data(2, 2, &image_data).unwrap(); let result = bitmap.set_id("id\0null"); - assert!(result.is_err()); + assert_eq!(result, Err(CString::new("id\0null").unwrap_err())); } } @@ -533,7 +526,7 @@ mod cstr_ptr_to_optional_string_tests { #[test] fn cstr_ptr_to_optional_string_returns_none_for_null() { - assert!(cstr_ptr_to_optional_string(ptr::null()).is_none()); + assert_eq!(cstr_ptr_to_optional_string(ptr::null()), None); } #[test] @@ -652,7 +645,7 @@ mod from_file_status_to_result_tests { } #[test] - fn from_file_status_to_result_maps_null_fname_arg() { + fn from_file_status_to_result_reports_a_null_fname_arg_as_a_contract_violation() { let outcome = unsafe { from_file_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_FNAME_ARG, @@ -674,7 +667,7 @@ mod from_file_status_to_result_tests { } #[test] - fn from_file_status_to_result_maps_null_out_bitmap_arg() { + fn from_file_status_to_result_reports_a_null_out_bitmap_arg_as_a_contract_violation() { let outcome = unsafe { from_file_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_OUT_BITMAP_ARG, @@ -696,7 +689,7 @@ mod from_file_status_to_result_tests { } #[test] - fn from_file_status_to_result_maps_llama_cpp_out_of_memory() { + fn from_file_status_to_result_reports_llama_cpp_running_out_of_memory() { let outcome = unsafe { from_file_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_LLAMA_CPP_OUT_OF_MEMORY, @@ -1047,35 +1040,38 @@ mod bitmap_init_status_to_result_tests { } #[test] - fn thrown_exceptions_report_their_messages() { + fn an_exception_thrown_while_initializing_an_image_bitmap_reports_its_message() { let image_error = unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"image threw".as_ptr()) }; + + assert_eq!( + bitmap_init_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + image_error, + ) + .err(), + Some(MtmdBitmapError::Reported { + message: "image threw".to_owned(), + }) + ); + } + + #[test] + fn an_exception_thrown_while_initializing_an_audio_bitmap_reports_its_message() { let audio_error = unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"audio threw".as_ptr()) }; assert_eq!( - [ - bitmap_init_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null_mut(), - image_error, - ) - .err(), - bitmap_init_from_audio_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null_mut(), - audio_error, - ) - .err(), - ], - [ - Some(MtmdBitmapError::Reported { - message: "image threw".to_owned(), - }), - Some(MtmdBitmapError::Reported { - message: "audio threw".to_owned(), - }), - ] + bitmap_init_from_audio_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + audio_error, + ) + .err(), + Some(MtmdBitmapError::Reported { + message: "audio threw".to_owned(), + }) ); } } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_context.rs b/llama-cpp-bindings/src/mtmd/mtmd_context.rs index 7685909a5..ec12ef9f7 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_context.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_context.rs @@ -499,7 +499,7 @@ mod map_tokenize_from_parts_status_tests { } #[test] - fn tokenize_from_parts_status_maps_preprocessing_or_tokenization_failure() { + fn tokenize_from_parts_status_reports_a_media_preprocessing_or_tokenization_failure() { assert_eq!( map_tokenize_from_parts_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_LLAMA_CPP_REPORTED_PREPROCESSING_OR_TOKENIZATION_FAILURE, @@ -535,7 +535,7 @@ mod map_tokenize_from_parts_status_tests { } #[test] - fn tokenize_from_parts_status_maps_llama_cpp_out_of_memory() { + fn tokenize_from_parts_status_reports_llama_cpp_running_out_of_memory() { assert_eq!( map_tokenize_from_parts_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_LLAMA_CPP_OUT_OF_MEMORY, @@ -570,7 +570,7 @@ mod map_tokenize_from_parts_status_tests { } #[test] - fn map_tokenize_from_parts_status_maps_llama_cpp_reported_malformed_part() { + fn map_tokenize_from_parts_status_reports_a_malformed_part_list_as_a_contract_violation() { assert_eq!( map_tokenize_from_parts_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_LLAMA_CPP_REPORTED_MALFORMED_PART, @@ -589,7 +589,7 @@ mod map_tokenize_from_parts_status_tests { } #[test] - fn map_tokenize_from_parts_status_maps_null_part_element() { + fn map_tokenize_from_parts_status_reports_a_null_part_element_as_a_contract_violation() { assert_eq!( map_tokenize_from_parts_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_NULL_PART_ELEMENT, @@ -608,7 +608,8 @@ mod map_tokenize_from_parts_status_tests { } #[test] - fn map_tokenize_from_parts_status_maps_llama_cpp_threw_cxx_exception_without_a_message() { + fn map_tokenize_from_parts_status_reports_an_exception_without_a_message_as_a_contract_violation() + { assert_eq!( map_tokenize_from_parts_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_LLAMA_CPP_THREW_CXX_EXCEPTION, @@ -627,7 +628,7 @@ mod map_tokenize_from_parts_status_tests { } #[test] - fn map_tokenize_from_parts_status_maps_null_ctx_arg() { + fn map_tokenize_from_parts_status_reports_a_null_ctx_arg_as_a_contract_violation() { assert_eq!( map_tokenize_from_parts_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_NULL_CTX_ARG, @@ -646,7 +647,7 @@ mod map_tokenize_from_parts_status_tests { } #[test] - fn map_tokenize_from_parts_status_maps_null_output_arg() { + fn map_tokenize_from_parts_status_reports_a_null_output_arg_as_a_contract_violation() { assert_eq!( map_tokenize_from_parts_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_NULL_OUTPUT_ARG, @@ -665,7 +666,8 @@ mod map_tokenize_from_parts_status_tests { } #[test] - fn map_tokenize_from_parts_status_maps_null_parts_arg_when_num_parts_nonzero() { + fn map_tokenize_from_parts_status_reports_a_null_parts_arg_with_a_nonzero_part_count_as_a_contract_violation() + { assert_eq!( map_tokenize_from_parts_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_NULL_PARTS_ARG_WHEN_NUM_PARTS_NONZERO, @@ -681,6 +683,10 @@ mod map_tokenize_from_parts_status_tests { .into() ) ); + } + + #[test] + fn map_tokenize_from_parts_status_preserves_an_unknown_status() { assert_eq!( map_tokenize_from_parts_status(255, 0, ptr::null_mut()).err(), Some(MtmdTokenizeError::FfiStatus(FfiStatusError { @@ -702,7 +708,7 @@ mod map_tokenize_status_tests { use crate::mtmd::mtmd_tokenize_error::MtmdTokenizeError; #[test] - fn tokenize_status_maps_bitmap_count_mismatch() { + fn tokenize_status_reports_a_bitmap_count_that_does_not_match_the_marker_count() { let result = map_tokenize_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_LLAMA_CPP_REPORTED_BITMAP_COUNT_DOES_NOT_MATCH_MARKER_COUNT, 0, @@ -716,7 +722,7 @@ mod map_tokenize_status_tests { } #[test] - fn tokenize_status_maps_preprocessing_or_tokenization_failure() { + fn tokenize_status_reports_a_media_preprocessing_or_tokenization_failure() { let result = map_tokenize_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_LLAMA_CPP_REPORTED_PREPROCESSING_OR_TOKENIZATION_FAILURE, 0, @@ -730,7 +736,7 @@ mod map_tokenize_status_tests { } #[test] - fn tokenize_status_maps_unknown_status_with_value() { + fn tokenize_status_maps_an_undocumented_code_with_its_value() { let result = map_tokenize_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_LLAMA_CPP_RETURNED_UNDOCUMENTED_NONZERO_CODE, 42, @@ -763,7 +769,7 @@ mod map_tokenize_status_tests { } #[test] - fn tokenize_status_maps_cxx_exception_to_without_a_message_is_a_contract_error() { + fn tokenize_status_reports_an_exception_without_a_message_as_a_contract_violation() { let result = map_tokenize_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_LLAMA_CPP_THREW_CXX_EXCEPTION, 0, @@ -811,7 +817,7 @@ mod map_tokenize_status_tests { } #[test] - fn map_tokenize_status_maps_null_ctx_arg() { + fn map_tokenize_status_reports_a_null_ctx_arg_as_a_contract_violation() { let outcome = map_tokenize_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_CTX_ARG, 0, @@ -830,7 +836,7 @@ mod map_tokenize_status_tests { } #[test] - fn map_tokenize_status_maps_null_output_arg() { + fn map_tokenize_status_reports_a_null_output_arg_as_a_contract_violation() { let outcome = map_tokenize_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_OUTPUT_ARG, 0, @@ -849,7 +855,7 @@ mod map_tokenize_status_tests { } #[test] - fn map_tokenize_status_maps_null_text_arg() { + fn map_tokenize_status_reports_a_null_text_arg_as_a_contract_violation() { let outcome = map_tokenize_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_TEXT_ARG, 0, @@ -868,7 +874,7 @@ mod map_tokenize_status_tests { } #[test] - fn map_tokenize_status_maps_llama_cpp_out_of_memory() { + fn map_tokenize_status_reports_llama_cpp_running_out_of_memory() { let outcome = map_tokenize_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_LLAMA_CPP_OUT_OF_MEMORY, 0, @@ -939,7 +945,7 @@ mod map_encode_chunk_status_tests { } #[test] - fn encode_chunk_status_maps_cxx_exception_to_without_a_message_is_a_contract_error() { + fn encode_chunk_status_reports_an_exception_without_a_message_as_a_contract_violation() { let result = map_encode_chunk_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_ENCODE_CHUNK_LLAMA_CPP_THREW_CXX_EXCEPTION, 0, @@ -970,7 +976,7 @@ mod map_encode_chunk_status_tests { } #[test] - fn map_encode_chunk_status_maps_null_ctx_arg() { + fn map_encode_chunk_status_reports_a_null_ctx_arg_as_a_contract_violation() { let outcome = map_encode_chunk_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_ENCODE_CHUNK_NULL_CTX_ARG, 0, @@ -989,7 +995,7 @@ mod map_encode_chunk_status_tests { } #[test] - fn map_encode_chunk_status_maps_null_chunk_arg() { + fn map_encode_chunk_status_reports_a_null_chunk_arg_as_a_contract_violation() { let outcome = map_encode_chunk_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_ENCODE_CHUNK_NULL_CHUNK_ARG, 0, @@ -1008,7 +1014,7 @@ mod map_encode_chunk_status_tests { } #[test] - fn map_encode_chunk_status_maps_llama_cpp_out_of_memory() { + fn map_encode_chunk_status_reports_llama_cpp_running_out_of_memory() { let outcome = map_encode_chunk_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_ENCODE_CHUNK_LLAMA_CPP_OUT_OF_MEMORY, 0, @@ -1076,7 +1082,7 @@ mod map_init_from_file_status_tests { } #[test] - fn init_from_file_status_maps_cxx_exception_to_without_a_message_is_a_contract_error() { + fn init_from_file_status_reports_an_exception_without_a_message_as_a_contract_violation() { let result = map_init_from_file_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), @@ -1109,7 +1115,7 @@ mod map_init_from_file_status_tests { } #[test] - fn map_init_from_file_status_maps_null_mmproj_path_arg() { + fn map_init_from_file_status_reports_a_null_mmproj_path_arg_as_a_contract_violation() { let outcome = map_init_from_file_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_NULL_MMPROJ_PATH_ARG, ptr::null_mut(), @@ -1129,7 +1135,7 @@ mod map_init_from_file_status_tests { } #[test] - fn map_init_from_file_status_maps_null_text_model_arg() { + fn map_init_from_file_status_reports_a_null_text_model_arg_as_a_contract_violation() { let outcome = map_init_from_file_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_NULL_TEXT_MODEL_ARG, ptr::null_mut(), @@ -1149,7 +1155,7 @@ mod map_init_from_file_status_tests { } #[test] - fn map_init_from_file_status_maps_null_out_ctx_arg() { + fn map_init_from_file_status_reports_a_null_out_ctx_arg_as_a_contract_violation() { let outcome = map_init_from_file_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_NULL_OUT_CTX_ARG, ptr::null_mut(), @@ -1169,7 +1175,7 @@ mod map_init_from_file_status_tests { } #[test] - fn map_init_from_file_status_maps_llama_cpp_out_of_memory() { + fn map_init_from_file_status_reports_llama_cpp_running_out_of_memory() { let outcome = map_init_from_file_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), @@ -1225,20 +1231,53 @@ mod capability_flags_tests { #[test] fn every_capability_of_llama_cpp_maps_to_its_own_flag() { - let all_capabilities = llama_cpp_bindings_sys::mtmd_helper_cap { - chat: true, - vision: true, - audio: true, - video: true, - audio_stream: true, - text_gen: true, - audio_gen: true, - audio_gen_spk_ref: true, - }; + let single_capabilities = [ + llama_cpp_bindings_sys::mtmd_helper_cap { + chat: true, + ..NO_CAPABILITIES + }, + llama_cpp_bindings_sys::mtmd_helper_cap { + vision: true, + ..NO_CAPABILITIES + }, + llama_cpp_bindings_sys::mtmd_helper_cap { + audio: true, + ..NO_CAPABILITIES + }, + llama_cpp_bindings_sys::mtmd_helper_cap { + video: true, + ..NO_CAPABILITIES + }, + llama_cpp_bindings_sys::mtmd_helper_cap { + audio_stream: true, + ..NO_CAPABILITIES + }, + llama_cpp_bindings_sys::mtmd_helper_cap { + text_gen: true, + ..NO_CAPABILITIES + }, + llama_cpp_bindings_sys::mtmd_helper_cap { + audio_gen: true, + ..NO_CAPABILITIES + }, + llama_cpp_bindings_sys::mtmd_helper_cap { + audio_gen_spk_ref: true, + ..NO_CAPABILITIES + }, + ]; assert_eq!( - capability_flags(all_capabilities), - BitFlags::::all() + single_capabilities.map(capability_flags), + [ + BitFlags::from_flag(MtmdCapability::Chat), + BitFlags::from_flag(MtmdCapability::Vision), + BitFlags::from_flag(MtmdCapability::Audio), + BitFlags::from_flag(MtmdCapability::Video), + BitFlags::from_flag(MtmdCapability::AudioStream), + BitFlags::from_flag(MtmdCapability::TextGeneration), + BitFlags::from_flag(MtmdCapability::AudioGeneration), + BitFlags::from_flag(MtmdCapability::AudioGenerationSpeakerReference), + ] ); } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs index 25b189efa..52e57c955 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs @@ -220,24 +220,22 @@ impl Drop for MtmdInputChunk { } #[cfg(test)] -mod unit_tests { +mod tokens_from_raw_ptr_tests { use std::ptr; - use super::non_causal_chunk_micro_batch_error; use super::tokens_from_raw_ptr; - use crate::mtmd::mtmd_eval_error::MtmdEvalError; - use crate::mtmd::non_causal_chunk_micro_batch_mismatch::NonCausalChunkMicroBatchMismatch; use crate::token::LlamaToken; #[test] fn tokens_from_raw_ptr_returns_none_for_null() { - assert!(unsafe { tokens_from_raw_ptr(ptr::null(), 5) }.is_none()); + assert_eq!(unsafe { tokens_from_raw_ptr(ptr::null(), 5) }, None); } #[test] fn tokens_from_raw_ptr_returns_none_for_zero_count() { let token: llama_cpp_bindings_sys::llama_token = 42; - assert!(unsafe { tokens_from_raw_ptr(&raw const token, 0) }.is_none()); + + assert_eq!(unsafe { tokens_from_raw_ptr(&raw const token, 0) }, None); } #[test] @@ -247,6 +245,13 @@ mod unit_tests { assert_eq!(result, Some(&[LlamaToken::new(1), LlamaToken::new(2)][..])); } +} + +#[cfg(test)] +mod non_causal_chunk_micro_batch_error_tests { + use super::non_causal_chunk_micro_batch_error; + use crate::mtmd::mtmd_eval_error::MtmdEvalError; + use crate::mtmd::non_causal_chunk_micro_batch_mismatch::NonCausalChunkMicroBatchMismatch; #[test] fn a_non_causal_chunk_larger_than_the_micro_batch_reports_the_mismatch() { @@ -263,12 +268,12 @@ mod unit_tests { #[test] fn a_causal_chunk_larger_than_the_micro_batch_is_split_instead() { - assert!(non_causal_chunk_micro_batch_error(false, 9, 4).is_none()); + assert_eq!(non_causal_chunk_micro_batch_error(false, 9, 4), None); } #[test] fn a_non_causal_chunk_within_the_micro_batch_fits() { - assert!(non_causal_chunk_micro_batch_error(true, 4, 4).is_none()); + assert_eq!(non_causal_chunk_micro_batch_error(true, 4, 4), None); } } diff --git a/llama-cpp-bindings/src/sampled_token.rs b/llama-cpp-bindings/src/sampled_token.rs index c3a6613c5..b93e50573 100644 --- a/llama-cpp-bindings/src/sampled_token.rs +++ b/llama-cpp-bindings/src/sampled_token.rs @@ -80,7 +80,7 @@ mod tests { } #[test] - fn a_token_of_no_section_is_undeterminable() { + fn a_token_of_the_pending_section_is_undeterminable() { assert_eq!( SampledToken::in_section(SampledTokenSection::Pending, LlamaToken::new(7)), SampledToken::Undeterminable(LlamaToken::new(7)) diff --git a/llama-cpp-bindings/src/sampling.rs b/llama-cpp-bindings/src/sampling.rs index e1c18106c..108e2ab41 100644 --- a/llama-cpp-bindings/src/sampling.rs +++ b/llama-cpp-bindings/src/sampling.rs @@ -978,7 +978,7 @@ impl Drop for LlamaSampler { #[cfg(test)] mod tests { - use std::ffi::CString; + use std::ffi::{CStr, CString}; use std::ptr; use std::slice; @@ -1065,7 +1065,7 @@ mod tests { } #[test] - fn null_native_sampler_is_initialization_error() { + fn from_raw_reports_a_null_native_sampler_as_unavailable() { let result = unsafe { LlamaSampler::from_raw(ptr::null_mut(), "test") }; assert_eq!( @@ -1086,7 +1086,7 @@ mod tests { } #[test] - fn sanitize_grammar_strings_null_byte_in_root_only() { + fn sanitize_grammar_strings_reports_a_nul_byte_in_the_root() { assert_eq!( LlamaSampler::sanitize_grammar_strings("root ::= \"hello\"", "ro\0ot"), Err(GrammarError::RootContainsNul( @@ -1096,7 +1096,7 @@ mod tests { } #[test] - fn sanitize_grammar_strings_null_byte_in_grammar() { + fn sanitize_grammar_strings_reports_a_nul_byte_in_the_grammar() { assert_eq!( LlamaSampler::sanitize_grammar_strings("root ::= \"\0\"", "root"), Err(GrammarError::GrammarContainsNul( @@ -1106,7 +1106,7 @@ mod tests { } #[test] - fn sanitize_grammar_strings_null_byte_in_root() { + fn sanitize_grammar_strings_reports_a_nul_byte_in_the_grammar_before_one_in_the_root() { assert_eq!( LlamaSampler::sanitize_grammar_strings("ro\0ot ::= \"hello\"", "ro\0ot"), Err(GrammarError::GrammarContainsNul( @@ -1119,26 +1119,25 @@ mod tests { #[test] fn sanitize_trigger_patterns_converts_every_pattern() { let patterns = vec!["^hello$".to_string(), "world.*".to_string()]; - let result = LlamaSampler::sanitize_trigger_patterns(&patterns); - assert!(result.is_ok()); - assert_eq!(result.expect("valid trigger patterns").len(), 2); + assert_eq!( + LlamaSampler::sanitize_trigger_patterns(&patterns), + Ok(vec![c"^hello$".to_owned(), c"world.*".to_owned()]) + ); } #[test] - fn sanitize_trigger_patterns_empty_list() { + fn sanitize_trigger_patterns_converts_an_empty_list_to_no_patterns() { let patterns: Vec = vec![]; - let result = LlamaSampler::sanitize_trigger_patterns(&patterns); - assert!(result.is_ok()); assert_eq!( - result.expect("valid trigger patterns"), - Vec::::new() + LlamaSampler::sanitize_trigger_patterns(&patterns), + Ok(Vec::::new()) ); } #[test] - fn sanitize_trigger_patterns_null_byte() { + fn sanitize_trigger_patterns_reports_a_nul_byte_in_a_pattern() { let patterns = vec!["hel\0lo".to_string()]; assert_eq!( LlamaSampler::sanitize_trigger_patterns(&patterns), @@ -1159,8 +1158,7 @@ mod tests { false, ); - assert!(sampler.apply(&mut data_array).is_ok()); - + assert_eq!(sampler.apply(&mut data_array), Ok(())); assert_eq!(data_array.selected_token(), Some(LlamaToken::new(1))); } @@ -1195,9 +1193,7 @@ mod tests { let mut sampler = LlamaSampler::chain([penalties_sampler(), greedy_sampler()]) .expect("sampler chain must initialize"); - sampler - .accept(LlamaToken::new(1)) - .expect("test: accept should succeed"); + assert_eq!(sampler.accept(LlamaToken::new(1)), Ok(())); } #[test] @@ -1205,39 +1201,73 @@ mod tests { let mut sampler = LlamaSampler::chain([penalties_sampler(), greedy_sampler()]) .expect("sampler chain must initialize"); - sampler - .accept_many([LlamaToken::new(1), LlamaToken::new(2), LlamaToken::new(3)]) - .expect("test: accept_many should succeed"); - } - - #[test] - fn every_sampler_constructor_initializes_its_sampler() { - let _temp = LlamaSampler::temp(0.8).expect("temperature sampler must initialize"); - let _temp_ext = LlamaSampler::temp_ext(0.8, 0.1, 1.0) - .expect("extended temperature sampler must initialize"); - let _top_k = LlamaSampler::top_k(40).expect("top-k sampler must initialize"); - let _top_n_sigma = - LlamaSampler::top_n_sigma(2.0).expect("top-n-sigma sampler must initialize"); - let _top_p = LlamaSampler::top_p(0.9, 1).expect("top-p sampler must initialize"); - let _min_p = LlamaSampler::min_p(0.05, 1).expect("min-p sampler must initialize"); - let _typical = LlamaSampler::typical(0.9, 1).expect("typical sampler must initialize"); - let _xtc = LlamaSampler::xtc(0.1, 0.5, 1, 42).expect("XTC sampler must initialize"); - let _dist = LlamaSampler::dist(42).expect("distribution sampler must initialize"); - let _mirostat = LlamaSampler::mirostat(32000, 42, 5.0, 0.1, 100) - .expect("Mirostat sampler must initialize"); - let _mirostat_v2 = - LlamaSampler::mirostat_v2(42, 5.0, 0.1).expect("Mirostat v2 sampler must initialize"); + assert_eq!( + sampler.accept_many([LlamaToken::new(1), LlamaToken::new(2), LlamaToken::new(3)]), + Ok(()) + ); + } + + #[test] + fn every_model_free_sampler_constructor_builds_the_matching_llama_cpp_sampler() { let biases = vec![LlamaLogitBias::new(LlamaToken::new(0), -100.0)]; - let _logit_bias = LlamaSampler::logit_bias(32000, &biases); - let _chain = - LlamaSampler::chain([greedy_sampler()]).expect("sampler chain must initialize"); + let samplers = [ + LlamaSampler::temp(0.8), + LlamaSampler::temp_ext(0.8, 0.1, 1.0), + LlamaSampler::top_k(40), + LlamaSampler::top_n_sigma(2.0), + LlamaSampler::top_p(0.9, 1), + LlamaSampler::min_p(0.05, 1), + LlamaSampler::typical(0.9, 1), + LlamaSampler::xtc(0.1, 0.5, 1, 42), + LlamaSampler::dist(42), + LlamaSampler::mirostat(32000, 42, 5.0, 0.1, 100), + LlamaSampler::mirostat_v2(42, 5.0, 0.1), + LlamaSampler::penalties(32_000, 64, 1.1, 0.0, 0.0), + LlamaSampler::logit_bias(32000, &biases), + LlamaSampler::greedy(), + LlamaSampler::chain([greedy_sampler()]), + ]; + + assert_eq!( + samplers.map(|initialized_sampler| { + initialized_sampler.map(|sampler| { + unsafe { + CStr::from_ptr(llama_cpp_bindings_sys::llama_sampler_name(sampler.as_ptr())) + } + .to_owned() + }) + }), + [ + Ok(c"temp".to_owned()), + Ok(c"temp-ext".to_owned()), + Ok(c"top-k".to_owned()), + Ok(c"top-n-sigma".to_owned()), + Ok(c"top-p".to_owned()), + Ok(c"min-p".to_owned()), + Ok(c"typical".to_owned()), + Ok(c"xtc".to_owned()), + Ok(c"dist".to_owned()), + Ok(c"mirostat".to_owned()), + Ok(c"mirostat-v2".to_owned()), + Ok(c"penalties".to_owned()), + Ok(c"logit-bias".to_owned()), + Ok(c"greedy".to_owned()), + Ok(c"chain".to_owned()), + ] + ); } #[test] - fn a_distribution_sampler_resets_and_reports_its_seed() { + fn resetting_a_sampler_succeeds_without_a_grammar_callback_failure() { let mut sampler = LlamaSampler::dist(42).expect("distribution sampler must initialize"); assert_eq!(sampler.reset(), Ok(())); + } + + #[test] + fn a_distribution_sampler_reports_the_seed_it_was_built_with() { + let sampler = LlamaSampler::dist(42).expect("distribution sampler must initialize"); + assert_eq!(sampler.get_seed(), 42); } @@ -1247,23 +1277,6 @@ mod tests { let debug_output = format!("{sampler:?}"); assert_eq!(debug_output, "LlamaSampler { .. }"); } -} - -#[cfg(test)] -mod checked_usize_as_i32_sampling_tests { - - use super::LlamaSampler; - use super::checked_usize_as_i32_sampling; - use crate::error::sampling_error::SamplingError; - use crate::token::logit_bias::LlamaLogitBias; - - #[test] - fn a_count_beyond_an_i32_is_an_integer_overflow() { - assert_eq!( - checked_usize_as_i32_sampling(usize::MAX), - i32::try_from(usize::MAX).map_err(SamplingError::IntegerOverflow) - ); - } #[test] fn a_logit_bias_list_longer_than_a_c_int_is_refused() { @@ -1288,7 +1301,7 @@ mod grammar_callback_error_to_result_tests { use crate::error::sample_error::SampleError; #[test] - fn grammar_callback_error_to_result_maps_recorded_error() { + fn grammar_callback_error_to_result_reports_the_recorded_grammar_callback_failure() { let result = grammar_callback_error_to_result(Some(RecordedError::new("mask failed".to_string()))); @@ -1301,8 +1314,8 @@ mod grammar_callback_error_to_result_tests { } #[test] - fn grammar_callback_error_to_result_maps_absence_to_ok() { - assert!(grammar_callback_error_to_result(None).is_ok()); + fn grammar_callback_error_to_result_succeeds_without_a_recorded_error() { + assert_eq!(grammar_callback_error_to_result(None), Ok(())); } } @@ -1314,7 +1327,7 @@ mod grammar_callback_error_to_accept_result_tests { use crate::error::sampler_accept_error::SamplerAcceptError; #[test] - fn grammar_callback_error_to_accept_result_maps_recorded_error() { + fn grammar_callback_error_to_accept_result_reports_the_recorded_grammar_callback_failure() { let result = grammar_callback_error_to_accept_result(Some(RecordedError::new( "consume failed".to_string(), ))); @@ -1328,8 +1341,8 @@ mod grammar_callback_error_to_accept_result_tests { } #[test] - fn grammar_callback_error_to_accept_result_maps_absence_to_ok() { - assert!(grammar_callback_error_to_accept_result(None).is_ok()); + fn grammar_callback_error_to_accept_result_succeeds_without_a_recorded_error() { + assert_eq!(grammar_callback_error_to_accept_result(None), Ok(())); } } @@ -1345,16 +1358,17 @@ mod check_sampler_accept_status_tests { #[test] fn check_sampler_accept_status_maps_ok_to_success() { - let result = check_sampler_accept_status( - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_OK, - ptr::null_mut(), + assert_eq!( + check_sampler_accept_status( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_OK, + ptr::null_mut(), + ), + Ok(()) ); - - assert!(result.is_ok()); } #[test] - fn check_sampler_accept_status_exception_maps_to_typed_variant() { + fn check_sampler_accept_status_reports_an_exception_as_a_corrupted_grammar_state() { let out_error = unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"grammar state corrupted".as_ptr()) }; @@ -1395,7 +1409,7 @@ mod check_sampler_accept_status_tests { } #[test] - fn check_sampler_accept_status_maps_null_out_error_arg() { + fn check_sampler_accept_status_reports_a_null_out_error_arg_as_a_contract_violation() { let outcome = check_sampler_accept_status( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_NULL_OUT_ERROR_ARG, ptr::null_mut(), @@ -1413,7 +1427,7 @@ mod check_sampler_accept_status_tests { } #[test] - fn check_sampler_accept_status_maps_llama_cpp_out_of_memory() { + fn check_sampler_accept_status_reports_llama_cpp_running_out_of_memory() { let outcome = check_sampler_accept_status( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), @@ -1436,7 +1450,7 @@ mod check_sampler_accept_status_tests { } #[test] - fn check_sampler_accept_status_maps_an_unknown_status() { + fn check_sampler_accept_status_preserves_an_unknown_status() { assert_eq!( check_sampler_accept_status( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_LLAMA_CPP_THREW_CXX_EXCEPTION + 1, @@ -1507,7 +1521,7 @@ mod sampler_sample_status_to_result_tests { } #[test] - fn sampler_sample_status_to_result_maps_null_sampler_arg() { + fn sampler_sample_status_to_result_reports_a_null_sampler_arg_as_a_contract_violation() { let outcome = sampler_sample_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_SAMPLER_ARG, 0, @@ -1526,7 +1540,7 @@ mod sampler_sample_status_to_result_tests { } #[test] - fn sampler_sample_status_to_result_maps_null_out_token_arg() { + fn sampler_sample_status_to_result_reports_a_null_out_token_arg_as_a_contract_violation() { let outcome = sampler_sample_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_OUT_TOKEN_ARG, 0, @@ -1545,7 +1559,7 @@ mod sampler_sample_status_to_result_tests { } #[test] - fn sampler_sample_status_to_result_maps_null_out_error_arg() { + fn sampler_sample_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() { let outcome = sampler_sample_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_OUT_ERROR_ARG, 0, @@ -1564,7 +1578,7 @@ mod sampler_sample_status_to_result_tests { } #[test] - fn sampler_sample_status_to_result_maps_llama_cpp_out_of_memory() { + fn sampler_sample_status_to_result_reports_llama_cpp_running_out_of_memory() { let outcome = sampler_sample_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_LLAMA_CPP_OUT_OF_MEMORY, 0, @@ -1591,7 +1605,7 @@ mod sampler_sample_status_to_result_tests { } #[test] - fn sampler_sample_status_to_result_maps_an_unknown_status() { + fn sampler_sample_status_to_result_preserves_an_unknown_status() { assert_eq!( sampler_sample_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_LLAMA_CPP_THREW_CXX_EXCEPTION + 1, @@ -1613,9 +1627,12 @@ mod sampler_sample_status_to_result_tests { mod sampler_init_status_to_result_tests { use std::ptr; + use llama_cpp_fault_injection::armed_fault::ArmedFault; + use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::LlamaSampler; use super::sampler_init_status_to_result; use crate::error::sampling_error::SamplingError; @@ -1686,7 +1703,7 @@ mod sampler_init_status_to_result_tests { } #[test] - fn sampler_init_status_to_result_maps_null_out_sampler_arg() { + fn sampler_init_status_to_result_reports_a_null_out_sampler_arg_as_a_contract_violation() { assert_eq!( sampler_init_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_NULL_OUT_SAMPLER_ARG, @@ -1706,7 +1723,7 @@ mod sampler_init_status_to_result_tests { } #[test] - fn sampler_init_status_to_result_maps_null_out_error_arg() { + fn sampler_init_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() { assert_eq!( sampler_init_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_NULL_OUT_ERROR_ARG, @@ -1726,7 +1743,8 @@ mod sampler_init_status_to_result_tests { } #[test] - fn sampler_init_status_to_result_maps_llama_cpp_threw_cxx_exception_without_a_message() { + fn sampler_init_status_to_result_reports_an_exception_without_a_message_as_a_contract_violation() + { assert_eq!( sampler_init_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, @@ -1746,7 +1764,7 @@ mod sampler_init_status_to_result_tests { } #[test] - fn sampler_init_status_to_result_maps_an_unknown_status() { + fn sampler_init_status_to_result_preserves_an_unknown_status() { assert_eq!( sampler_init_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION + 1, @@ -1767,6 +1785,38 @@ mod sampler_init_status_to_result_tests { ) ); } + + #[test] + fn initializing_a_selecting_sampler_reports_the_exception_llama_cpp_throws() { + let distribution_failure = { + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_INIT_DIST_THROWS, + ); + + LlamaSampler::dist(7).err() + }; + let greedy_failure = { + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GREEDY_THROWS, + ); + + LlamaSampler::greedy().err() + }; + + assert_eq!( + [distribution_failure, greedy_failure], + [ + Some(SamplingError::Reported { + sampler: "distribution", + message: injected_fault_message(), + }), + Some(SamplingError::Reported { + sampler: "greedy", + message: injected_fault_message(), + }), + ] + ); + } } #[cfg(test)] @@ -1819,7 +1869,7 @@ mod sampler_chain_add_status_to_result_tests { } #[test] - fn sampler_chain_add_status_to_result_maps_null_chain_arg() { + fn sampler_chain_add_status_to_result_reports_a_null_chain_arg_as_a_contract_violation() { assert_eq!( sampler_chain_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_NULL_CHAIN_ARG, @@ -1834,7 +1884,7 @@ mod sampler_chain_add_status_to_result_tests { } #[test] - fn sampler_chain_add_status_to_result_maps_null_sampler_arg() { + fn sampler_chain_add_status_to_result_reports_a_null_sampler_arg_as_a_contract_violation() { assert_eq!( sampler_chain_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_NULL_SAMPLER_ARG, @@ -1849,7 +1899,7 @@ mod sampler_chain_add_status_to_result_tests { } #[test] - fn sampler_chain_add_status_to_result_maps_null_out_error_arg() { + fn sampler_chain_add_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() { assert_eq!( sampler_chain_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_NULL_OUT_ERROR_ARG, @@ -1864,7 +1914,8 @@ mod sampler_chain_add_status_to_result_tests { } #[test] - fn sampler_chain_add_status_to_result_maps_llama_cpp_threw_cxx_exception_without_a_message() { + fn sampler_chain_add_status_to_result_reports_an_exception_without_a_message_as_a_contract_violation() + { assert_eq!( sampler_chain_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION, @@ -1879,7 +1930,7 @@ mod sampler_chain_add_status_to_result_tests { } #[test] - fn sampler_chain_add_status_to_result_maps_an_unknown_status() { + fn sampler_chain_add_status_to_result_preserves_an_unknown_status() { assert_eq!( sampler_chain_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION @@ -1964,7 +2015,8 @@ mod sampler_init_grammar_status_to_result_tests { } #[test] - fn sampler_init_grammar_status_to_result_maps_null_out_error_arg() { + fn sampler_init_grammar_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() + { let outcome = sampler_init_grammar_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_NULL_OUT_ERROR_ARG, ptr::null_mut(), @@ -1983,7 +2035,7 @@ mod sampler_init_grammar_status_to_result_tests { } #[test] - fn sampler_init_grammar_status_to_result_maps_llama_cpp_out_of_memory() { + fn sampler_init_grammar_status_to_result_reports_llama_cpp_running_out_of_memory() { let outcome = sampler_init_grammar_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), @@ -2011,7 +2063,7 @@ mod sampler_init_grammar_status_to_result_tests { } #[test] - fn sampler_init_grammar_status_to_result_maps_an_unknown_status() { + fn sampler_init_grammar_status_to_result_preserves_an_unknown_status() { assert_eq!( sampler_init_grammar_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LLAMA_CPP_THREW_CXX_EXCEPTION @@ -2119,7 +2171,8 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { } #[test] - fn sampler_init_grammar_lazy_patterns_status_to_result_maps_null_out_error_arg() { + fn sampler_init_grammar_lazy_patterns_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() + { let outcome = sampler_init_grammar_lazy_patterns_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_NULL_OUT_ERROR_ARG, ptr::null_mut(), @@ -2138,7 +2191,8 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { } #[test] - fn sampler_init_grammar_lazy_patterns_status_to_result_maps_llama_cpp_out_of_memory() { + fn sampler_init_grammar_lazy_patterns_status_to_result_reports_llama_cpp_running_out_of_memory() + { let outcome = sampler_init_grammar_lazy_patterns_status_to_result(llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), ptr::null_mut()); assert_eq!(outcome.err(), Some(GrammarError::LlamaCppOutOfMemory)); } @@ -2162,7 +2216,7 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { } #[test] - fn sampler_init_grammar_lazy_patterns_status_to_result_maps_an_unknown_status() { + fn sampler_init_grammar_lazy_patterns_status_to_result_preserves_an_unknown_status() { let unknown_status = llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_LLAMA_CPP_THREW_CXX_EXCEPTION + 1; @@ -2193,7 +2247,6 @@ mod sampler_application_tests { use super::LlamaSampler; use crate::error::sample_error::SampleError; use crate::error::sampler_apply_error::SamplerApplyError; - use crate::error::sampling_error::SamplingError; use crate::token::LlamaToken; use crate::token::data::LlamaTokenData; use crate::token::data_array::LlamaTokenDataArray; @@ -2268,38 +2321,6 @@ mod sampler_application_tests { })) ); } - - #[test] - fn selecting_samplers_llama_cpp_cannot_build_are_reported() { - let distribution_failure = { - let _fault = ArmedFault::at( - llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_INIT_DIST_THROWS, - ); - - LlamaSampler::dist(7).err() - }; - let greedy_failure = { - let _fault = ArmedFault::at( - llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GREEDY_THROWS, - ); - - LlamaSampler::greedy().err() - }; - - assert_eq!( - [distribution_failure, greedy_failure], - [ - Some(SamplingError::Reported { - sampler: "distribution", - message: injected_fault_message(), - }), - Some(SamplingError::Reported { - sampler: "greedy", - message: injected_fault_message(), - }), - ] - ); - } } #[cfg(test)] @@ -2352,7 +2373,7 @@ mod sampler_apply_status_to_result_tests { } #[test] - fn sampler_apply_status_to_result_maps_null_sampler_arg() { + fn sampler_apply_status_to_result_reports_a_null_sampler_arg_as_a_null_sampler() { let outcome = sampler_apply_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_SAMPLER_ARG, ptr::null_mut(), @@ -2361,7 +2382,7 @@ mod sampler_apply_status_to_result_tests { } #[test] - fn sampler_apply_status_to_result_maps_null_data_array_arg() { + fn sampler_apply_status_to_result_reports_a_null_data_array_arg_as_a_contract_violation() { let outcome = sampler_apply_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_DATA_ARRAY_ARG, ptr::null_mut(), @@ -2379,7 +2400,7 @@ mod sampler_apply_status_to_result_tests { } #[test] - fn sampler_apply_status_to_result_maps_null_out_error_arg() { + fn sampler_apply_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() { let outcome = sampler_apply_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_OUT_ERROR_ARG, ptr::null_mut(), @@ -2397,7 +2418,7 @@ mod sampler_apply_status_to_result_tests { } #[test] - fn sampler_apply_status_to_result_maps_llama_cpp_out_of_memory() { + fn sampler_apply_status_to_result_reports_llama_cpp_running_out_of_memory() { let outcome = sampler_apply_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), diff --git a/llama-cpp-bindings/src/send_logs_to_log.rs b/llama-cpp-bindings/src/send_logs_to_log.rs index 89cd19980..87f964670 100644 --- a/llama-cpp-bindings/src/send_logs_to_log.rs +++ b/llama-cpp-bindings/src/send_logs_to_log.rs @@ -186,7 +186,6 @@ mod tests { use llama_cpp_log_decoder::log_level::LogLevel; use llama_cpp_log_decoder::log_line::LogLine; - use super::GGML_SOURCE; use super::LLAMA_SOURCE; use super::LogSource; use super::dispatch_output; @@ -252,6 +251,13 @@ mod tests { .collect() } + fn records_with_text(target: &str, text: &str) -> Vec { + records_for(target) + .into_iter() + .filter(|record| record.text == text) + .collect() + } + fn invoke_callback( level: llama_cpp_bindings_sys::ggml_log_level, text: &CStr, @@ -482,12 +488,33 @@ mod tests { #[test] #[serial] - fn sending_logs_to_log_installs_both_log_sources() { + fn sending_logs_to_log_creates_the_llama_cpp_source_and_routes_ggml_logs() { ensure_test_logger_installed(); send_logs_to_log(LogOptions::default()); - assert!(LLAMA_SOURCE.get().is_some()); - assert!(GGML_SOURCE.get().is_some()); + let mut ggml_callback: llama_cpp_bindings_sys::ggml_log_callback = None; + let mut ggml_user_data: *mut c_void = ptr::null_mut(); + + unsafe { + llama_cpp_bindings_sys::ggml_log_get(&raw mut ggml_callback, &raw mut ggml_user_data); + ggml_callback.unwrap()( + llama_cpp_bindings_sys::GGML_LOG_LEVEL_WARN, + c"routed through ggml\n".as_ptr(), + ggml_user_data, + ); + } + + assert_eq!( + LLAMA_SOURCE.get().map(|llama_source| llama_source.target), + Some("llama.cpp") + ); + assert_eq!( + records_with_text("ggml", "routed through ggml"), + [LogRecord { + level: Level::Warn, + text: "routed through ggml".to_owned(), + }] + ); } #[test] diff --git a/llama-cpp-bindings/src/streaming_json_probe.rs b/llama-cpp-bindings/src/streaming_json_probe.rs index a57f56cb4..8bf511a08 100644 --- a/llama-cpp-bindings/src/streaming_json_probe.rs +++ b/llama-cpp-bindings/src/streaming_json_probe.rs @@ -85,7 +85,7 @@ mod tests { } #[test] - fn partial_name_key_is_still_possibly_valid() { + fn a_name_key_awaiting_its_colon_is_still_possibly_valid() { assert_eq!(probe(r#"{ "name""#), JsonProbeOutcome::StillPossiblyValid,); } @@ -169,14 +169,6 @@ mod tests { ); } - #[test] - fn complete_tool_call_with_internal_whitespace_is_completed_valid() { - assert_eq!( - probe(r#"{"name": "f", "arguments": {}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - #[test] fn complete_tool_call_with_string_argument_is_completed_valid() { assert_eq!( @@ -185,14 +177,6 @@ mod tests { ); } - #[test] - fn complete_tool_call_with_multiple_arguments_is_completed_valid() { - assert_eq!( - probe(r#"{"name":"book_flight","arguments":{"from":"NYC","to":"PAR","passengers":2}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - #[test] fn complete_tool_call_with_nested_arguments_is_completed_valid() { assert_eq!( @@ -233,14 +217,6 @@ mod tests { ); } - #[test] - fn complete_tool_call_with_array_inside_arguments_is_completed_valid() { - assert_eq!( - probe(r#"{"name":"f","arguments":{"items":[1,2,3]}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - #[test] fn complete_tool_call_without_arguments_field_is_completed_valid() { assert_eq!( @@ -250,22 +226,12 @@ mod tests { } #[test] - fn top_level_array_is_failed() { + fn a_top_level_value_other_than_an_object_is_failed() { assert_eq!(probe("["), JsonProbeOutcome::Failed); } #[test] - fn top_level_scalar_number_is_failed() { - assert_eq!(probe("123"), JsonProbeOutcome::Failed); - } - - #[test] - fn top_level_string_is_failed() { - assert_eq!(probe(r#""hi""#), JsonProbeOutcome::Failed); - } - - #[test] - fn complete_object_with_wrong_first_key_is_failed() { + fn an_unknown_key_is_failed() { assert_eq!(probe(r#"{"foo":"bar"}"#), JsonProbeOutcome::Failed,); } @@ -275,7 +241,7 @@ mod tests { } #[test] - fn complete_object_with_non_string_name_is_failed() { + fn a_name_value_that_is_not_a_string_is_failed() { assert_eq!( probe(r#"{"name":123,"arguments":{}}"#), JsonProbeOutcome::Failed, @@ -283,37 +249,13 @@ mod tests { } #[test] - fn complete_object_with_null_name_is_failed() { - assert_eq!( - probe(r#"{"name":null,"arguments":{}}"#), - JsonProbeOutcome::Failed, - ); - } - - #[test] - fn complete_object_with_arguments_as_array_is_failed() { + fn an_arguments_value_that_is_not_an_object_is_failed() { assert_eq!( probe(r#"{"name":"f","arguments":[]}"#), JsonProbeOutcome::Failed, ); } - #[test] - fn complete_object_with_arguments_as_string_is_failed() { - assert_eq!( - probe(r#"{"name":"f","arguments":"hi"}"#), - JsonProbeOutcome::Failed, - ); - } - - #[test] - fn complete_object_with_third_top_level_key_is_failed() { - assert_eq!( - probe(r#"{"name":"f","arguments":{},"extra":1}"#), - JsonProbeOutcome::Failed, - ); - } - #[test] fn complete_object_with_empty_name_is_failed() { assert_eq!( @@ -322,14 +264,6 @@ mod tests { ); } - #[test] - fn complete_object_with_trailing_garbage_is_failed() { - assert_eq!( - probe(r#"{"name":"f","arguments":{}}garbage"#), - JsonProbeOutcome::Failed, - ); - } - #[test] fn empty_object_is_failed_due_to_missing_required_name() { assert_eq!(probe("{}"), JsonProbeOutcome::Failed); @@ -354,7 +288,7 @@ mod tests { } #[test] - fn complete_tool_call_followed_by_second_object_is_failed() { + fn a_second_object_after_the_closed_tool_call_is_failed() { assert_eq!( probe(r#"{"name":"a","arguments":{}}{"name":"b","arguments":{}}"#), JsonProbeOutcome::Failed, @@ -362,36 +296,12 @@ mod tests { } #[test] - fn buffer_with_only_open_quote_is_still_possibly_valid() { + fn a_partially_written_key_is_still_possibly_valid() { assert_eq!(probe(r#"{ "n"#), JsonProbeOutcome::StillPossiblyValid,); } #[test] - fn buffer_with_complete_first_field_unknown_second_key_is_failed() { - assert_eq!( - probe(r#"{ "name": "f", "foo": 1}"#), - JsonProbeOutcome::Failed, - ); - } - - #[test] - fn unicode_letter_inside_name_value_completes_validly() { - assert_eq!( - probe(r#"{"name":"éclair","arguments":{}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn arguments_field_with_explicit_null_is_failed() { - assert_eq!( - probe(r#"{"name":"f","arguments":null}"#), - JsonProbeOutcome::Failed, - ); - } - - #[test] - fn syntactically_malformed_object_is_failed() { + fn a_comma_before_the_first_key_is_failed() { assert_eq!(probe("{,}"), JsonProbeOutcome::Failed,); } @@ -427,9 +337,17 @@ mod tests { #[test] fn syntax_error_inside_arguments_fails_when_the_object_closes() { + let mut streaming_probe = StreamingJsonProbe::default(); + assert_eq!( - probe(r#"{"name":"f","arguments":{"q" 1}}"#), - JsonProbeOutcome::Failed, + [ + streaming_probe.feed(r#"{"name":"f","arguments":{"q" 1}"#), + streaming_probe.feed("}"), + ], + [ + JsonProbeOutcome::StillPossiblyValid, + JsonProbeOutcome::Failed + ] ); } diff --git a/llama-cpp-bindings/src/token/data_array.rs b/llama-cpp-bindings/src/token/data_array.rs index fd1afbb27..a3285225f 100644 --- a/llama-cpp-bindings/src/token/data_array.rs +++ b/llama-cpp-bindings/src/token/data_array.rs @@ -124,7 +124,7 @@ mod tests { false, ); - assert!(array.selected_token().is_none()); + assert_eq!(array.selected_token(), None); } #[test] @@ -135,7 +135,7 @@ mod tests { sorted: false, }; - assert!(array.selected_token().is_none()); + assert_eq!(array.selected_token(), None); } #[test] diff --git a/llama-cpp-bindings/src/tool_call_format.rs b/llama-cpp-bindings/src/tool_call_format.rs index 2e231e0ae..d023dac7e 100644 --- a/llama-cpp-bindings/src/tool_call_format.rs +++ b/llama-cpp-bindings/src/tool_call_format.rs @@ -62,7 +62,6 @@ mod tests { use super::ToolCallFormatOutcome; use super::try_parse; - use crate::chat_template_tool_calls::known_marker_candidates::known_marker_candidates; use crate::error::bracketed_args_failure::BracketedArgsFailure; use crate::error::tool_call_format_failure::ToolCallFormatFailure; @@ -234,155 +233,4 @@ mod tests { ToolCallFormatOutcome::NoMatch, ); } - - #[test] - fn try_parse_returns_no_match_for_plain_content_under_every_known_shape() { - let plain_content = "Sorry, I cannot help with that request."; - - for candidate in known_marker_candidates() { - assert_eq!( - try_parse(plain_content, &candidate), - ToolCallFormatOutcome::NoMatch, - "expected NoMatch for plain content under candidate {candidate:?}" - ); - } - } - - #[test] - fn duck_type_resolves_qwen_xml_input_via_xml_tags_shape_first() { - let qwen_input = "\n\ - \n\ - \n\ - Paris\n\ - \n\ - \n\ - "; - - let mut resolved = None; - for candidate in known_marker_candidates() { - if let ToolCallFormatOutcome::Parsed(calls) = try_parse(qwen_input, &candidate) { - resolved = Some((candidate.args_shape, calls)); - break; - } - } - - let (args_shape, calls) = - resolved.expect("Qwen XML input must resolve via at least one duck-type candidate"); - assert_eq!( - args_shape, - ToolCallArgsShape::XmlTags(XmlTagsShape { - function_open_prefix: "".to_owned(), - parameter_open_prefix: "".to_owned(), - }), - "duck-type ordering must resolve Qwen XML via the XmlTags shape (most restrictive \ - shape that requires `".to_owned(), - key_close: "".to_owned(), - value_open: "".to_owned(), - value_close: "".to_owned(), - }), - "GLM input must resolve via the KeyValueXmlTags shape, got {args_shape:?}" - ); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].name, "get_weather"); - assert_eq!( - calls[0].arguments, - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ); - } - - #[test] - fn duck_type_resolves_mistral_input_via_bracketed_json_shape() { - let mistral_input = r#"[TOOL_CALLS]get_weather[ARGS]{"location":"Paris"}"#; - - let mut resolved = None; - for candidate in known_marker_candidates() { - if let ToolCallFormatOutcome::Parsed(calls) = try_parse(mistral_input, &candidate) { - resolved = Some((candidate.args_shape, calls)); - break; - } - } - - let (args_shape, calls) = - resolved.expect("Mistral input must resolve via at least one duck-type candidate"); - assert_eq!( - args_shape, - ToolCallArgsShape::BracketedJson(BracketedJsonShape { - name_args_separator: "[ARGS]".to_owned(), - }), - "Mistral input must resolve via the BracketedJson shape; the candidate ordering must \ - try BracketedJson before PairedQuote because PairedQuote's `{{` separator could \ - greedily match Mistral's JSON args. Got {args_shape:?}" - ); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].name, "get_weather"); - assert_eq!( - calls[0].arguments, - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ); - } - - #[test] - fn duck_type_resolves_gemma_input_via_paired_quote_shape() { - let gemma_input = "<|tool_call>call:get_weather{location:<|\"|>Paris<|\"|>}"; - - let mut resolved = None; - for candidate in known_marker_candidates() { - if let ToolCallFormatOutcome::Parsed(calls) = try_parse(gemma_input, &candidate) { - resolved = Some((candidate.args_shape, calls)); - break; - } - } - - let (args_shape, calls) = - resolved.expect("Gemma input must resolve via at least one duck-type candidate"); - assert_eq!( - args_shape, - ToolCallArgsShape::PairedQuote(PairedQuoteShape { - name_args_separator: "{".to_owned(), - value_quote: ToolCallValueQuote { - open: "<|\"|>".to_owned(), - close: "<|\"|>".to_owned(), - }, - }), - "Gemma input must resolve via the PairedQuote shape, got {args_shape:?}" - ); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].name, "get_weather"); - assert_eq!( - calls[0].arguments, - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ); - } } diff --git a/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs b/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs index c8d57dcee..cd9022342 100644 --- a/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs +++ b/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs @@ -105,8 +105,6 @@ pub fn parse( #[cfg(test)] mod tests { - use std::mem::discriminant; - use serde_json::json; use llama_cpp_bindings_types::bracketed_json_shape::BracketedJsonShape; @@ -191,26 +189,22 @@ mod tests { } #[test] - fn rejects_malformed_json_arguments_with_typed_failure() { - let result = parse( - "[TOOL_CALLS]get_weather[ARGS]{\"location\":}", - &mistral3_markers(), - &mistral3_shape(), - ); - - let failure = result.expect_err("malformed JSON must produce a typed failure"); - + fn malformed_json_arguments_are_reported_with_the_parser_message() { assert_eq!( - discriminant(&failure), - discriminant(&BracketedArgsFailure::InvalidJsonArguments { - tool_name: String::new(), - message: String::new(), - }), + parse( + "[TOOL_CALLS]get_weather[ARGS]{\"location\":}", + &mistral3_markers(), + &mistral3_shape(), + ), + Err(BracketedArgsFailure::InvalidJsonArguments { + tool_name: "get_weather".to_owned(), + message: "expected value at line 1 column 13".to_owned(), + }) ); } #[test] - fn rejects_truncated_json_arguments_with_unterminated_failure() { + fn whitespace_only_arguments_are_reported_as_unterminated() { let failure = parse( "[TOOL_CALLS]get_weather[ARGS] ", &mistral3_markers(), diff --git a/llama-cpp-bindings/src/tool_call_format/json_object.rs b/llama-cpp-bindings/src/tool_call_format/json_object.rs index 3b2d75a2e..a05193831 100644 --- a/llama-cpp-bindings/src/tool_call_format/json_object.rs +++ b/llama-cpp-bindings/src/tool_call_format/json_object.rs @@ -165,23 +165,19 @@ mod tests { } #[test] - fn rejects_input_without_open_brace() { + fn input_without_an_open_brace_parses_no_tool_calls() { let parsed = parse("plain content", &qwen3_shape()).expect("must parse"); assert_eq!(parsed, Vec::::new()); } #[test] - fn rejects_array_instead_of_object() { - let parsed = parse("[1, 2, 3]", &qwen3_shape()).expect("must parse"); - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn returns_failure_for_malformed_json() { - let err = parse(r#"{"name": "f", "arguments": {"a": }"#, &qwen3_shape()).unwrap_err(); - let JsonObjectFailure::InvalidJson { message } = err; - - assert_ne!(message, ""); + fn malformed_json_is_reported_with_the_parser_message() { + assert_eq!( + parse(r#"{"name": "f", "arguments": {"a": }"#, &qwen3_shape()), + Err(JsonObjectFailure::InvalidJson { + message: "expected value at line 1 column 34".to_owned(), + }) + ); } #[test] diff --git a/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs b/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs index 4156b735e..27d5765ce 100644 --- a/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs +++ b/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs @@ -326,17 +326,6 @@ mod tests { ); } - #[test] - fn parses_bare_numeric_value() { - let parsed = - parse(&gemma4_call("a:42"), &gemma4_markers(), &gemma4_shape()).expect("must parse"); - - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"a": 42})), - ); - } - #[test] fn parses_comma_separated_bare_values() { let parsed = parse( @@ -355,17 +344,6 @@ mod tests { ); } - #[test] - fn parses_bare_boolean_value() { - let parsed = - parse(&gemma4_call("a:true"), &gemma4_markers(), &gemma4_shape()).expect("must parse"); - - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"a": true})), - ); - } - #[test] fn rejects_unclosed_quoted_value_with_typed_failure() { let result = parse( diff --git a/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs b/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs index 4ffd31624..88db0f5d4 100644 --- a/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs +++ b/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs @@ -323,12 +323,6 @@ mod tests { assert_eq!(parsed, Vec::::new()); } - #[test] - fn returns_empty_for_empty_body() { - let parsed = parse("", &xml_shape()).expect("must parse empty"); - assert_eq!(parsed, Vec::::new()); - } - #[test] fn returns_empty_when_shape_has_empty_required_field() { let mut shape = xml_shape(); @@ -339,7 +333,7 @@ mod tests { } #[test] - fn parses_negotiate_with_cat_reproducer_payload() { + fn parses_newline_padded_parameters_inside_a_tool_call_block_into_typed_arguments() { let body = "\n\ \n\ \n\ diff --git a/llama-cpp-error-recorder/src/error_scope.rs b/llama-cpp-error-recorder/src/error_scope.rs index ef018c2e1..7c21d7e40 100644 --- a/llama-cpp-error-recorder/src/error_scope.rs +++ b/llama-cpp-error-recorder/src/error_scope.rs @@ -37,17 +37,6 @@ mod tests { use crate::record::record; use crate::recorded_error::RecordedError; - #[test] - fn records_and_takes_within_a_scope() { - let scope = ErrorScope::enter(); - record(RecordedError::new("boom".to_string())); - - assert_eq!( - scope.take().map(RecordedError::into_message), - Some("boom".to_string()) - ); - } - #[test] fn keeps_the_first_recorded_error() { let scope = ErrorScope::enter(); @@ -72,7 +61,10 @@ mod tests { let scope = ErrorScope::enter(); record(RecordedError::new("once".to_string())); - assert!(scope.take().is_some()); + assert_eq!( + scope.take().map(RecordedError::into_message), + Some("once".to_string()) + ); assert!(scope.take().is_none()); } diff --git a/llama-cpp-gbnf/src/validate_gbnf.rs b/llama-cpp-gbnf/src/validate_gbnf.rs index db255f476..ca48794e3 100644 --- a/llama-cpp-gbnf/src/validate_gbnf.rs +++ b/llama-cpp-gbnf/src/validate_gbnf.rs @@ -197,7 +197,7 @@ mod tests { } #[test] - fn exception_status_without_message_maps_to_unknown_reported_error() { + fn exception_status_without_message_is_a_contract_error() { assert_eq!( validation_status_to_result( LLAMA_RS_GBNF_VALIDATION_THREW_CXX_EXCEPTION, @@ -276,7 +276,7 @@ mod tests { } #[test] - fn llama_cpp_out_of_memory_status_is_distinct_from_error_string_allocation() { + fn llama_cpp_out_of_memory_status_maps_to_llama_cpp_out_of_memory() { assert_eq!( validation_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_LLAMA_CPP_OUT_OF_MEMORY, diff --git a/llama-cpp-log-decoder/src/log_decoder.rs b/llama-cpp-log-decoder/src/log_decoder.rs index 0b5284acb..a4116af76 100644 --- a/llama-cpp-log-decoder/src/log_decoder.rs +++ b/llama-cpp-log-decoder/src/log_decoder.rs @@ -193,24 +193,6 @@ mod tests { ); } - #[test] - fn a_continuation_completes_the_held_line() { - let mut decoder = LogDecoder::new(); - decoder.feed(IncomingLogLevel::Info, "hello "); - let result = decoder.feed(IncomingLogLevel::Cont, "world\n"); - - assert_eq!( - result, - DecodeResult { - output: DecodeOutput::Line(LogLine { - level: LogLevel::Info, - text: "hello world".to_owned(), - }), - anomaly: None, - } - ); - } - #[test] fn continuations_join_until_the_newline() { let mut decoder = LogDecoder::new(); diff --git a/llama-cpp-test-harness-macros/src/lib.rs b/llama-cpp-test-harness-macros/src/lib.rs index 83b02b9fe..7d2b4b6f6 100644 --- a/llama-cpp-test-harness-macros/src/lib.rs +++ b/llama-cpp-test-harness-macros/src/lib.rs @@ -52,7 +52,7 @@ mod tests { } #[test] - fn dispatch_on_valid_input_emits_inventory_submission() { + fn dispatch_on_valid_input_emits_the_expanded_tokens() { let attribute = quote! { model_source = HuggingFace("r", "f"), n_gpu_layers = 0, diff --git a/llama-cpp-test-harness-macros/src/parsed_args.rs b/llama-cpp-test-harness-macros/src/parsed_args.rs index 8cef8f5b4..bb2306c7a 100644 --- a/llama-cpp-test-harness-macros/src/parsed_args.rs +++ b/llama-cpp-test-harness-macros/src/parsed_args.rs @@ -4,9 +4,11 @@ use proc_macro2::Ident; use proc_macro2::Span; use syn::Expr; use syn::ExprLit; +use syn::ExprUnary; use syn::Lit; use syn::Meta; use syn::Token; +use syn::UnOp; use syn::parse::Parse; use syn::parse::ParseStream; use syn::punctuated::Punctuated; @@ -79,6 +81,37 @@ fn require_i32_lit(literal: &Lit, field: &str) -> syn::Result { } } +fn require_negated_i32_lit(literal: &Lit, field: &str) -> syn::Result { + if let Lit::Int(int_literal) = literal { + format!("-{}", int_literal.base10_digits()) + .parse::() + .map_err(|parse_error| { + syn::Error::new_spanned( + literal, + format!("field `{field}` expects an integer that fits in i32: {parse_error}"), + ) + }) + } else { + Err(syn::Error::new_spanned( + literal, + format!("field `{field}` expects an integer literal"), + )) + } +} + +fn require_i32_value(expression: &Expr, field: &str) -> syn::Result { + if let Expr::Unary(ExprUnary { + op: UnOp::Neg(_), + expr: negated, + .. + }) = expression + { + require_negated_i32_lit(literal_from_expression(negated)?, field) + } else { + require_i32_lit(literal_from_expression(expression)?, field) + } +} + fn require_bool_lit(literal: &Lit, field: &str) -> syn::Result { if let Lit::Bool(bool_literal) = literal { Ok(bool_literal.value()) @@ -108,10 +141,7 @@ fn dispatch_field( accumulator.mmproj_source = Some(ParsedSource::parse(value, "mmproj_source")?); } "n_gpu_layers" => { - accumulator.n_gpu_layers = Some(require_i32_lit( - literal_from_expression(value)?, - "n_gpu_layers", - )?); + accumulator.n_gpu_layers = Some(require_i32_value(value, "n_gpu_layers")?); } "n_ctx" => { accumulator.n_ctx = Some(require_int_lit(literal_from_expression(value)?, "n_ctx")?); @@ -154,10 +184,7 @@ fn dispatch_field( )?); } "n_threads_batch" => { - accumulator.n_threads_batch = Some(require_i32_lit( - literal_from_expression(value)?, - "n_threads_batch", - )?); + accumulator.n_threads_batch = Some(require_i32_value(value, "n_threads_batch")?); } "void_logs" => { accumulator.void_logs = Some(require_bool_lit( @@ -408,71 +435,117 @@ mod tests { } #[test] - fn non_literal_value_for_scalar_field_is_rejected() { + fn load_mode_literal_is_rejected() { let source = "\ model_source = HuggingFace(\"x\", \"y\"), \ - n_gpu_layers = some_const, \ - load_mode = Mmap, \ + n_gpu_layers = 0, \ + load_mode = \"Mmap\", \ n_ctx = 1, \ n_batch = 1, \ n_ubatch = 1"; let message = parse(source) - .expect_err("non-literal value must fail") + .expect_err("literal load mode must fail") .to_string(); - assert_eq!(message, "expected a literal (string, integer, or bool)"); + assert_eq!(message, "expected a load-mode identifier"); } #[test] - fn wrong_literal_kind_for_int_field_is_rejected() { + fn qualified_load_mode_path_is_rejected() { let source = "\ model_source = HuggingFace(\"x\", \"y\"), \ - n_gpu_layers = \"nine\", \ - load_mode = Mmap, \ + n_gpu_layers = 0, \ + load_mode = modes::Mmap, \ n_ctx = 1, \ n_batch = 1, \ n_ubatch = 1"; let message = parse(source) - .expect_err("string for int field must fail") + .expect_err("qualified load mode must fail") .to_string(); - assert_eq!(message, "field `n_gpu_layers` expects an integer literal"); + assert_eq!(message, "expected a load-mode identifier"); } #[test] - fn load_mode_literal_is_rejected() { + fn negative_i32_fields_are_parsed() { let source = "\ model_source = HuggingFace(\"x\", \"y\"), \ - n_gpu_layers = 0, \ - load_mode = \"Mmap\", \ + n_gpu_layers = -1, \ + load_mode = Mmap, \ + n_ctx = 1, \ + n_batch = 1, \ + n_ubatch = 1, \ + n_threads_batch = -2"; + let parsed = parse(source).expect("negative i32 fields must parse"); + + assert_eq!( + [ + parsed.model_load_params.n_gpu_layers, + parsed + .context_params + .n_threads_batch + .expect("n_threads_batch was given") + ], + [-1, -2] + ); + } + + #[test] + fn a_negative_i32_field_below_i32_min_is_rejected() { + let source = "\ + model_source = HuggingFace(\"x\", \"y\"), \ + n_gpu_layers = -2147483649, \ + load_mode = Mmap, \ n_ctx = 1, \ n_batch = 1, \ n_ubatch = 1"; - let message = parse(source) - .expect_err("literal load mode must fail") - .to_string(); - assert_eq!(message, "expected a load-mode identifier"); + assert_eq!( + parse(source) + .expect_err("an i32 below i32::MIN must fail") + .to_string(), + "field `n_gpu_layers` expects an integer that fits in i32: number too small to fit in target type" + ); } #[test] - fn qualified_load_mode_path_is_rejected() { + fn a_negated_i32_field_that_is_not_an_integer_is_rejected() { let source = "\ model_source = HuggingFace(\"x\", \"y\"), \ - n_gpu_layers = 0, \ - load_mode = modes::Mmap, \ + n_gpu_layers = -1.5, \ + load_mode = Mmap, \ n_ctx = 1, \ n_batch = 1, \ n_ubatch = 1"; - let message = parse(source) - .expect_err("qualified load mode must fail") - .to_string(); - assert_eq!(message, "expected a load-mode identifier"); + assert_eq!( + parse(source) + .expect_err("a negated float must fail") + .to_string(), + "field `n_gpu_layers` expects an integer literal" + ); + } + + #[test] + fn a_negated_i32_field_that_is_not_a_literal_is_rejected() { + let source = "\ + model_source = HuggingFace(\"x\", \"y\"), \ + n_gpu_layers = -layers, \ + load_mode = Mmap, \ + n_ctx = 1, \ + n_batch = 1, \ + n_ubatch = 1"; + + assert_eq!( + parse(source) + .expect_err("a negated identifier must fail") + .to_string(), + "expected a literal (string, integer, or bool)" + ); } #[test] - fn negative_int_for_u32_field_is_rejected() { + fn negative_int_for_u32_field_is_rejected_as_a_non_literal() { let source = "\ model_source = HuggingFace(\"x\", \"y\"), \ n_gpu_layers = 0, \ @@ -630,14 +703,6 @@ mod tests { assert!(parsed.context_params.embeddings); } - #[test] - fn optional_embeddings_false_is_parsed() { - let source = format!("{ALL_REQUIRED}, embeddings = false"); - let parsed = parse(&source).expect("embeddings = false must parse"); - - assert!(!parsed.context_params.embeddings); - } - #[test] fn optional_embeddings_rejects_non_bool_literal() { let source = format!("{ALL_REQUIRED}, embeddings = 1"); @@ -855,11 +920,10 @@ mod tests { #[test] fn unparseable_attribute_token_stream_is_rejected() { - let result = parse("@&^!"); + let message = parse("@&^!") + .expect_err("garbage attribute body must fail") + .to_string(); - assert!( - result.is_err(), - "garbage attribute body must fail to parse as Punctuated" - ); + assert_eq!(message, "expected identifier"); } } diff --git a/llama-cpp-test-harness-macros/src/parsed_source.rs b/llama-cpp-test-harness-macros/src/parsed_source.rs index 49d9c1ed1..0e56675de 100644 --- a/llama-cpp-test-harness-macros/src/parsed_source.rs +++ b/llama-cpp-test-harness-macros/src/parsed_source.rs @@ -182,7 +182,7 @@ mod tests { } #[test] - fn non_string_argument_is_rejected() { + fn huggingface_with_non_string_first_argument_is_rejected() { let message = parse("HuggingFace(42, \"file\")") .expect_err("non-string arg must fail") .to_string(); @@ -217,16 +217,6 @@ mod tests { ); } - #[test] - fn unparseable_input_returns_err() { - let result = parse("@&^!"); - - assert!( - result.is_err(), - "garbage input must fail to parse as syn::Expr" - ); - } - #[test] fn non_path_function_expression_is_rejected() { let message = parse("(closure)(\"a\")") diff --git a/llama-cpp-test-harness/src/context_params.rs b/llama-cpp-test-harness/src/context_params.rs index e135b9770..304f00b0e 100644 --- a/llama-cpp-test-harness/src/context_params.rs +++ b/llama-cpp-test-harness/src/context_params.rs @@ -59,52 +59,18 @@ mod tests { n_ctx: 1024, n_batch: 256, n_ubatch: 128, - ..BASELINE + n_seq_max: 4, + n_threads_batch: Some(8), + embeddings: true, } .into_llama_context_params(); assert_eq!(params.n_ctx(), NonZeroU32::new(1024)); assert_eq!(params.n_batch(), 256); assert_eq!(params.n_ubatch(), 128); - } - - #[test] - fn into_llama_context_params_propagates_embeddings_flag() { - let off = ContextParams { - embeddings: false, - ..BASELINE - } - .into_llama_context_params(); - let on = ContextParams { - embeddings: true, - ..BASELINE - } - .into_llama_context_params(); - - assert!(!off.embeddings()); - assert!(on.embeddings()); - } - - #[test] - fn into_llama_context_params_propagates_n_seq_max() { - let params = ContextParams { - n_seq_max: 4, - ..BASELINE - } - .into_llama_context_params(); - - assert_eq!(params.context_params.n_seq_max, 4); - } - - #[test] - fn into_llama_context_params_applies_n_threads_batch_when_some() { - let params = ContextParams { - n_threads_batch: Some(8), - ..BASELINE - } - .into_llama_context_params(); - - assert_eq!(params.context_params.n_threads_batch, 8); + assert_eq!(params.n_seq_max(), 4); + assert_eq!(params.n_threads_batch(), 8); + assert!(params.embeddings()); } #[test] diff --git a/llama-cpp-test-harness/src/model_load_params.rs b/llama-cpp-test-harness/src/model_load_params.rs index 5ee9e2433..6e2be6ef0 100644 --- a/llama-cpp-test-harness/src/model_load_params.rs +++ b/llama-cpp-test-harness/src/model_load_params.rs @@ -27,7 +27,7 @@ mod tests { use super::ModelLoadParams; #[test] - fn into_llama_model_params_carries_all_three_fields() { + fn into_llama_model_params_carries_both_fields() { let params = ModelLoadParams { n_gpu_layers: 7, load_mode: LlamaLoadMode::Mlock, diff --git a/llama-cpp-test-harness/src/run.rs b/llama-cpp-test-harness/src/run.rs index f1e9e0ed3..435e5ce89 100644 --- a/llama-cpp-test-harness/src/run.rs +++ b/llama-cpp-test-harness/src/run.rs @@ -29,16 +29,12 @@ mod tests { use std::process::ExitCode; use std::sync::PoisonError; - use libtest_mimic::Arguments; use libtest_mimic::Conclusion; - use llama_cpp_bindings::error::llama_cpp_error::LlamaCppError; use llama_cpp_bindings::llama_backend::LlamaBackend; use super::aggregate_exit_code; use super::run; - use crate::harness_run_error::HarnessRunError; - use crate::run_to_conclusions::run_to_conclusions; use crate::test_backend_gate::BACKEND_INIT_GATE; fn passing_conclusion() -> Conclusion { @@ -92,23 +88,6 @@ mod tests { assert_eq!(as_u8(code), 0); } - #[test] - fn run_to_conclusions_errors_when_backend_init_fails() { - let _gate = BACKEND_INIT_GATE - .lock() - .unwrap_or_else(PoisonError::into_inner); - let _hold = LlamaBackend::init().expect("first init must succeed"); - - let outcome = run_to_conclusions(Arguments::default()); - - assert_eq!( - outcome.err(), - Some(HarnessRunError::BackendInit( - LlamaCppError::BackendAlreadyInitialized - )) - ); - } - #[test] fn run_returns_exit_code_two_when_backend_init_fails() { let _gate = BACKEND_INIT_GATE diff --git a/llama-cpp-test-harness/src/run_to_conclusions.rs b/llama-cpp-test-harness/src/run_to_conclusions.rs index 8d2e3cf4f..97118959f 100644 --- a/llama-cpp-test-harness/src/run_to_conclusions.rs +++ b/llama-cpp-test-harness/src/run_to_conclusions.rs @@ -33,6 +33,9 @@ mod tests { use libtest_mimic::Arguments; + use llama_cpp_bindings::error::llama_cpp_error::LlamaCppError; + use llama_cpp_bindings::llama_backend::LlamaBackend; + use super::run_to_conclusions; use crate::harness_arguments_error::HarnessArgumentsError; use crate::harness_run_error::HarnessRunError; @@ -53,17 +56,26 @@ mod tests { } #[test] - fn empty_inventory_yields_no_conclusions_and_skips_void_logs() { + fn an_already_initialized_backend_stops_the_run_with_a_backend_init_error() { let _gate = BACKEND_INIT_GATE .lock() .unwrap_or_else(PoisonError::into_inner); + let _hold = LlamaBackend::init().expect("first init must succeed"); - let conclusions = run_to_conclusions(Arguments::default()) - .expect("empty inventory must run without a setup failure"); - - assert!( - conclusions.is_empty(), - "empty inventory must yield zero phase conclusions" + assert_eq!( + run_to_conclusions(Arguments::default()).err(), + Some(HarnessRunError::BackendInit( + LlamaCppError::BackendAlreadyInitialized + )) ); } + + #[test] + fn empty_inventory_yields_no_conclusions() { + let _gate = BACKEND_INIT_GATE + .lock() + .unwrap_or_else(PoisonError::into_inner); + + assert_eq!(run_to_conclusions(Arguments::default()), Ok(Vec::new())); + } } diff --git a/llama-cpp-test-harness/tests/phase_conclusions.rs b/llama-cpp-test-harness/tests/phase_conclusions.rs index 28c61c24f..a2fcffa02 100644 --- a/llama-cpp-test-harness/tests/phase_conclusions.rs +++ b/llama-cpp-test-harness/tests/phase_conclusions.rs @@ -18,7 +18,7 @@ use llama_cpp_test_harness_macros::llama_test; n_ubatch = 64, void_logs = true, )] -fn phase_a_first_passing_trial(fixture: &LlamaFixture<'_>) { +fn phase_a_passing_trial_that_requests_void_logs(fixture: &LlamaFixture<'_>) { let formatted = format!("{:?}", fixture.model); assert!(formatted.contains("LlamaModel")); no_op(fixture); @@ -114,7 +114,7 @@ fn phase_b_second_passing_trial(fixture: &LlamaFixture<'_>) { n_batch = 1, n_ubatch = 1 )] -fn shared_setup_failure_and_phase_a_trial(fixture: &LlamaFixture<'_>) { +fn phase_a_trial_also_scheduled_into_every_setup_failure_phase(fixture: &LlamaFixture<'_>) { assert!(fixture.model_path.exists()); } @@ -127,7 +127,7 @@ fn shared_setup_failure_and_phase_a_trial(fixture: &LlamaFixture<'_>) { n_ubatch = 64, mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf") )] -fn phase_d_mmproj_trial(fixture: &LlamaFixture<'_>) { +fn phase_d_passing_trial_with_a_loaded_mmproj(fixture: &LlamaFixture<'_>) { assert!( fixture.mtmd_context.is_some(), "mmproj_file declared, but fixture.mtmd_context is None", diff --git a/llama-cpp-wrapper-sources/src/compile_commands_file.rs b/llama-cpp-wrapper-sources/src/compile_commands_file.rs index a4ffb3374..6e0cc8c44 100644 --- a/llama-cpp-wrapper-sources/src/compile_commands_file.rs +++ b/llama-cpp-wrapper-sources/src/compile_commands_file.rs @@ -80,7 +80,6 @@ mod tests { use crate::system_include_flags::system_include_flags; use crate::wrapper_include_dirs::WRAPPER_INCLUDE_DIRS; use crate::wrapper_source_paths::wrapper_source_paths; - use crate::wrapper_sources_error::WrapperSourcesError; fn database_for(output_path: &str) -> CompileCommandsFile { CompileCommandsFile { @@ -136,15 +135,18 @@ mod tests { assert_eq!( commands - .first() + .iter() .map(|command| command .arguments .iter() .filter(|argument| argument.starts_with("-isystem")) .cloned() .collect::>()) - .unwrap_or_default(), - system_include_flags(Path::new("/repo/llama-cpp-bindings-sys")) + .collect::>>(), + vec![ + system_include_flags(Path::new("/repo/llama-cpp-bindings-sys")); + wrapper_source_paths(Path::new("/repo/llama-cpp-bindings-sys")).len() + ] ); } @@ -172,20 +174,32 @@ mod tests { #[test] fn creating_the_database_under_a_missing_directory_reports_the_path() { - let output_path = "/nonexistent-directory/compile_commands.json"; - - assert!(matches!( - database_for(output_path).write(), - Err(WrapperSourcesError::Create { ref path, .. }) if path == Path::new(output_path) - )); + assert_eq!( + database_for("/nonexistent-directory/compile_commands.json") + .write() + .err() + .map(|error| error.to_string()), + Some( + "compilation database file /nonexistent-directory/compile_commands.json could \ + not be created: No such file or directory (os error 2)" + .to_owned() + ) + ); } #[cfg(target_os = "linux")] #[test] fn a_destination_that_cannot_absorb_the_bytes_reports_the_path() { - assert!(matches!( - database_for("/dev/full").write(), - Err(WrapperSourcesError::Serialize { ref path, .. }) if path == Path::new("/dev/full") - )); + assert_eq!( + database_for("/dev/full") + .write() + .err() + .map(|error| error.to_string()), + Some( + "compilation database could not be written to /dev/full: No space left on \ + device (os error 28)" + .to_owned() + ) + ); } } diff --git a/llama-cpp-wrapper-sources/src/wrapper_sources_response_file.rs b/llama-cpp-wrapper-sources/src/wrapper_sources_response_file.rs index e3c5cc419..5a34274ed 100644 --- a/llama-cpp-wrapper-sources/src/wrapper_sources_response_file.rs +++ b/llama-cpp-wrapper-sources/src/wrapper_sources_response_file.rs @@ -31,23 +31,24 @@ impl WrapperSourcesResponseFile { #[cfg(test)] mod tests { - use std::path::Path; use std::path::PathBuf; use super::WrapperSourcesResponseFile; - use crate::wrapper_sources_error::WrapperSourcesError; #[test] fn writing_under_a_missing_directory_reports_the_path() { - let output_path = "/nonexistent-directory/wrapper_sources.rsp"; let response_file = WrapperSourcesResponseFile { - output_path: PathBuf::from(output_path), + output_path: PathBuf::from("/nonexistent-directory/wrapper_sources.rsp"), sys_dir: PathBuf::from("/repo/llama-cpp-bindings-sys"), }; - assert!(matches!( - response_file.write(), - Err(WrapperSourcesError::Write { ref path, .. }) if path == Path::new(output_path) - )); + assert_eq!( + response_file.write().err().map(|error| error.to_string()), + Some( + "wrapper source list could not be written to \ + /nonexistent-directory/wrapper_sources.rsp: No such file or directory (os error 2)" + .to_owned() + ) + ); } } diff --git a/llama-cpp-wrapper-sources/tests/emit_compile_commands.rs b/llama-cpp-wrapper-sources/tests/emit_compile_commands.rs index f4c729008..4c2678ba1 100644 --- a/llama-cpp-wrapper-sources/tests/emit_compile_commands.rs +++ b/llama-cpp-wrapper-sources/tests/emit_compile_commands.rs @@ -56,7 +56,7 @@ fn the_binary_fails_when_the_database_cannot_be_written() { } #[test] -fn the_binary_requires_every_output_path() { +fn the_binary_requires_every_argument() { let database_path = temporary_path("unwritten.json"); assert!(!emitter_succeeded(&[])); From 1197b71a5e9a78208f8c9dc5c46effa0a6efa435 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Sun, 1 Nov 2026 05:40:51 +0100 Subject: [PATCH 38/82] Make test helpers panic or return typed errors instead of untested error paths --- .../src/logits_left_by.rs | 17 ++++++----- llama-cpp-bindings-tests/src/single_token.rs | 20 ++++++------- .../src/two_sequence_context.rs | 9 +++--- .../tests/context_creation_limits.rs | 7 +---- .../tests/detokenization.rs | 7 ++--- .../tests/grammar_sampler.rs | 6 ++-- .../tests/lazy_grammar_sampler.rs | 10 +++---- .../tests/llguidance_sampler.rs | 30 +++++++++---------- .../tests/logit_bias_sampler.rs | 2 +- .../src/parsed_source.rs | 3 +- llama-cpp-test-harness/src/llama_fixture.rs | 9 +++--- 11 files changed, 56 insertions(+), 64 deletions(-) diff --git a/llama-cpp-bindings-tests/src/logits_left_by.rs b/llama-cpp-bindings-tests/src/logits_left_by.rs index d69b0b93a..519f4a5bc 100644 --- a/llama-cpp-bindings-tests/src/logits_left_by.rs +++ b/llama-cpp-bindings-tests/src/logits_left_by.rs @@ -1,13 +1,12 @@ -use anyhow::Result; - use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings::token::LlamaToken; use llama_cpp_bindings::token::data::LlamaTokenData; use llama_cpp_bindings::token::data_array::LlamaTokenDataArray; -/// # Errors -/// Forwards [`LlamaSampler::apply`] errors verbatim. -pub fn logits_left_by(sampler: &LlamaSampler, candidates: &[LlamaToken]) -> Result> { +/// # Panics +/// Panics when `sampler` cannot be applied to the candidates. +#[must_use] +pub fn logits_left_by(sampler: &LlamaSampler, candidates: &[LlamaToken]) -> Vec { let mut token_data_array = LlamaTokenDataArray::new( candidates .iter() @@ -16,11 +15,13 @@ pub fn logits_left_by(sampler: &LlamaSampler, candidates: &[LlamaToken]) -> Resu false, ); - sampler.apply(&mut token_data_array)?; + sampler + .apply(&mut token_data_array) + .expect("the sampler must apply to the candidates"); - Ok(token_data_array + token_data_array .data .iter() .map(LlamaTokenData::logit) - .collect()) + .collect() } diff --git a/llama-cpp-bindings-tests/src/single_token.rs b/llama-cpp-bindings-tests/src/single_token.rs index 0c9aac2fd..f7588aab9 100644 --- a/llama-cpp-bindings-tests/src/single_token.rs +++ b/llama-cpp-bindings-tests/src/single_token.rs @@ -1,17 +1,17 @@ -use anyhow::Result; -use anyhow::bail; - use llama_cpp_bindings::model::LlamaModel; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::token::LlamaToken; -/// # Errors -/// Forwards tokenization errors, and fails when `text` tokenizes into anything but one token. -pub fn single_token(model: &LlamaModel, text: &str) -> Result { - let [token] = model.str_to_token(text, AddBos::Never, ParseSpecialTokens::Always)?[..] else { - bail!("{text:?} must be a single token"); - }; +/// # Panics +/// Panics when `text` cannot be tokenized or does not tokenize into exactly one token. +#[must_use] +pub fn single_token(model: &LlamaModel, text: &str) -> LlamaToken { + let [token]: [LlamaToken; 1] = model + .str_to_token(text, AddBos::Never, ParseSpecialTokens::Always) + .expect("the text must tokenize") + .try_into() + .expect("the text must tokenize into exactly one token"); - Ok(token) + token } diff --git a/llama-cpp-bindings-tests/src/two_sequence_context.rs b/llama-cpp-bindings-tests/src/two_sequence_context.rs index 1eef43cd1..9d3955e04 100644 --- a/llama-cpp-bindings-tests/src/two_sequence_context.rs +++ b/llama-cpp-bindings-tests/src/two_sequence_context.rs @@ -1,19 +1,18 @@ -use anyhow::Result; - use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::error::llama_context_load_error::LlamaContextLoadError; use llama_cpp_test_harness::llama_fixture::LlamaFixture; /// # Errors /// Forwards [`LlamaContext::from_model`] errors verbatim. pub fn two_sequence_context<'model>( fixture: &LlamaFixture<'model>, -) -> Result> { - Ok(LlamaContext::from_model( +) -> Result, LlamaContextLoadError> { + LlamaContext::from_model( fixture.model, fixture.backend, (*fixture.context_params) .into_llama_context_params() .with_kv_unified(true) .with_n_seq_max(2), - )?) + ) } diff --git a/llama-cpp-bindings-tests/tests/context_creation_limits.rs b/llama-cpp-bindings-tests/tests/context_creation_limits.rs index 05d919230..1293ddd8b 100644 --- a/llama-cpp-bindings-tests/tests/context_creation_limits.rs +++ b/llama-cpp-bindings-tests/tests/context_creation_limits.rs @@ -41,12 +41,7 @@ use llama_cpp_test_harness_macros::llama_test; )] fn a_context_size_overflowing_llama_cpp_padding_is_unconstructible(fixture: &LlamaFixture<'_>) { assert_eq!( - LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - ) - .err(), + fixture.build_context().err(), Some(LlamaContextLoadError::Unconstructible) ); } diff --git a/llama-cpp-bindings-tests/tests/detokenization.rs b/llama-cpp-bindings-tests/tests/detokenization.rs index 235140ffa..6529b28f2 100644 --- a/llama-cpp-bindings-tests/tests/detokenization.rs +++ b/llama-cpp-bindings-tests/tests/detokenization.rs @@ -261,11 +261,9 @@ fn token_to_piece_strips_the_requested_count_of_leading_spaces( n_batch = 128, n_ubatch = 64, )] -fn token_to_piece_decodes_a_token_alike_in_every_non_content_section( - fixture: &LlamaFixture<'_>, -) -> Result<()> { +fn token_to_piece_decodes_a_token_alike_in_every_non_content_section(fixture: &LlamaFixture<'_>) { let model = fixture.model; - let hi_token = single_token(model, "hi")?; + let hi_token = single_token(model, "hi"); assert_eq!( [ @@ -287,7 +285,6 @@ fn token_to_piece_decodes_a_token_alike_in_every_non_content_section( Ok("hi".to_owned()) ] ); - Ok(()) } #[llama_test( diff --git a/llama-cpp-bindings-tests/tests/grammar_sampler.rs b/llama-cpp-bindings-tests/tests/grammar_sampler.rs index 3c76529e4..25f07574e 100644 --- a/llama-cpp-bindings-tests/tests/grammar_sampler.rs +++ b/llama-cpp-bindings-tests/tests/grammar_sampler.rs @@ -216,8 +216,8 @@ fn a_grammar_sampler_masks_tokens_that_cannot_start_the_grammar( assert_eq!( logits_left_by( &sampler, - &[single_token(model, "hello")?, single_token(model, "yes")?] - )?, + &[single_token(model, "hello"), single_token(model, "yes")] + ), [0.0, f32::NEG_INFINITY] ); @@ -236,7 +236,7 @@ fn qwen35_grammar_with_a_token_reference_constrains_the_first_token( fixture: &LlamaFixture<'_>, ) -> Result<()> { let model = fixture.model; - let think_token = single_token(model, "")?; + let think_token = single_token(model, ""); let mut context = LlamaContext::from_model( model, fixture.backend, diff --git a/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs b/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs index e8a873618..4e1b0c8e1 100644 --- a/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs +++ b/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs @@ -27,8 +27,8 @@ fn a_lazy_grammar_awaiting_its_trigger_pattern_leaves_candidates_unconstrained( assert_eq!( logits_left_by( &sampler, - &[single_token(model, "hello")?, single_token(model, "yes")?] - )?, + &[single_token(model, "hello"), single_token(model, "yes")] + ), [0.0, 0.0] ); @@ -89,14 +89,14 @@ fn a_lazy_grammar_awaiting_its_trigger_token_leaves_candidates_unconstrained( "root ::= \"hello\"", "root", &[], - &[single_token(model, "{")?], + &[single_token(model, "{")], )?; assert_eq!( logits_left_by( &sampler, - &[single_token(model, "hello")?, single_token(model, "yes")?] - )?, + &[single_token(model, "hello"), single_token(model, "yes")] + ), [0.0, 0.0] ); diff --git a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs index c2c43ca6c..f7903f51a 100644 --- a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs +++ b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs @@ -40,12 +40,12 @@ fn out_of_vocabulary_failure() -> String { .to_string() } -fn yes_no_and_hello(model: &LlamaModel) -> Result> { - Ok(vec![ - single_token(model, "yes")?, - single_token(model, "no")?, - single_token(model, "hello")?, - ]) +fn yes_no_and_hello(model: &LlamaModel) -> [LlamaToken; 3] { + [ + single_token(model, "yes"), + single_token(model, "no"), + single_token(model, "hello"), + ] } #[llama_test( @@ -89,8 +89,8 @@ fn a_json_schema_sampler_masks_tokens_that_cannot_start_the_schema( assert_eq!( logits_left_by( &sampler, - &[single_token(model, "{")?, single_token(model, "hello")?] - )?, + &[single_token(model, "{"), single_token(model, "hello")] + ), [0.0, f32::NEG_INFINITY] ); @@ -144,7 +144,7 @@ fn a_regex_sampler_masks_tokens_that_cannot_start_a_match( let sampler = LlamaSampler::llguidance(model, "regex", REGEX_GRAMMAR)?; assert_eq!( - logits_left_by(&sampler, &yes_no_and_hello(model)?)?, + logits_left_by(&sampler, &yes_no_and_hello(model)), [0.0, 0.0, f32::NEG_INFINITY] ); @@ -190,7 +190,7 @@ fn a_lark_sampler_masks_tokens_that_cannot_start_the_grammar( let sampler = create_llg_sampler(model, "lark", LARK_GRAMMAR)?; assert_eq!( - logits_left_by(&sampler, &yes_no_and_hello(model)?)?, + logits_left_by(&sampler, &yes_no_and_hello(model)), [0.0, 0.0, f32::NEG_INFINITY] ); @@ -409,8 +409,8 @@ fn a_sampler_cloned_through_ffi_keeps_its_own_grammar_state( ) }?; - assert_eq!(cloned.accept(single_token(model, "yes")?), Ok(())); - assert_eq!(sampler.accept(single_token(model, "no")?), Ok(())); + assert_eq!(cloned.accept(single_token(model, "yes")), Ok(())); + assert_eq!(sampler.accept(single_token(model, "no")), Ok(())); Ok(()) } @@ -600,13 +600,13 @@ fn samplers_sharing_the_cached_tokenizer_environment_mask_alike( fixture: &LlamaFixture<'_>, ) -> Result<()> { let model = fixture.model; - let candidates = yes_no_and_hello(model)?; + let candidates = yes_no_and_hello(model); let first = create_llg_sampler(model, "regex", REGEX_GRAMMAR)?; let second = create_llg_sampler(model, "regex", REGEX_GRAMMAR)?; assert_eq!( - logits_left_by(&first, &candidates)?, - logits_left_by(&second, &candidates)? + logits_left_by(&first, &candidates), + logits_left_by(&second, &candidates) ); Ok(()) diff --git a/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs b/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs index 1dcdfe945..3f7c9e052 100644 --- a/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs +++ b/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs @@ -18,7 +18,7 @@ fn a_logit_bias_sampler_with_an_empty_bias_list_leaves_candidates_unchanged( _fixture: &LlamaFixture<'_>, ) -> Result<()> { assert_eq!( - logits_left_by(&LlamaSampler::logit_bias(0, &[])?, &[LlamaToken::new(0)])?, + logits_left_by(&LlamaSampler::logit_bias(0, &[])?, &[LlamaToken::new(0)]), [0.0] ); diff --git a/llama-cpp-test-harness-macros/src/parsed_source.rs b/llama-cpp-test-harness-macros/src/parsed_source.rs index 0e56675de..e2a07e308 100644 --- a/llama-cpp-test-harness-macros/src/parsed_source.rs +++ b/llama-cpp-test-harness-macros/src/parsed_source.rs @@ -106,7 +106,8 @@ mod tests { use super::ParsedSource; fn parse(source: &str) -> syn::Result { - let expression: syn::Expr = parse_str(source)?; + let expression: syn::Expr = + parse_str(source).expect("the test source must be an expression"); ParsedSource::parse(&expression, "model_source") } diff --git a/llama-cpp-test-harness/src/llama_fixture.rs b/llama-cpp-test-harness/src/llama_fixture.rs index 5e0598d12..463ca61cc 100644 --- a/llama-cpp-test-harness/src/llama_fixture.rs +++ b/llama-cpp-test-harness/src/llama_fixture.rs @@ -1,8 +1,7 @@ use std::path::Path; -use anyhow::Result; - use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::error::llama_context_load_error::LlamaContextLoadError; use llama_cpp_bindings::llama_backend::LlamaBackend; use llama_cpp_bindings::model::LlamaModel; use llama_cpp_bindings::mtmd::mtmd_context::MtmdContext; @@ -20,11 +19,11 @@ pub struct LlamaFixture<'fixture> { impl LlamaFixture<'_> { /// # Errors /// Forwards [`LlamaContext::from_model`] errors verbatim. - pub fn build_context(&self) -> Result> { - Ok(LlamaContext::from_model( + pub fn build_context(&self) -> Result, LlamaContextLoadError> { + LlamaContext::from_model( self.model, self.backend, (*self.context_params).into_llama_context_params(), - )?) + ) } } From 68380e7b34c65efe17ee46a447286e74e51df104 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 06:55:22 +0100 Subject: [PATCH 39/82] Remove synchronize and detach_threadpool, which nothing in the bindings can observe --- .../tests/context_synchronization.rs | 25 ------------------- llama-cpp-bindings-tests/tests/main.rs | 2 -- .../tests/threadpool_detachment.rs | 25 ------------------- llama-cpp-bindings/src/context.rs | 8 ------ 4 files changed, 60 deletions(-) delete mode 100644 llama-cpp-bindings-tests/tests/context_synchronization.rs delete mode 100644 llama-cpp-bindings-tests/tests/threadpool_detachment.rs diff --git a/llama-cpp-bindings-tests/tests/context_synchronization.rs b/llama-cpp-bindings-tests/tests/context_synchronization.rs deleted file mode 100644 index 435d3c84d..000000000 --- a/llama-cpp-bindings-tests/tests/context_synchronization.rs +++ /dev/null @@ -1,25 +0,0 @@ -use anyhow::Result; - -use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_test_harness::llama_fixture::LlamaFixture; -use llama_cpp_test_harness_macros::llama_test; - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn an_idle_context_synchronizes(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - context.synchronize(); - - Ok(()) -} diff --git a/llama-cpp-bindings-tests/tests/main.rs b/llama-cpp-bindings-tests/tests/main.rs index b683012f1..e1a6ab49d 100644 --- a/llama-cpp-bindings-tests/tests/main.rs +++ b/llama-cpp-bindings-tests/tests/main.rs @@ -10,7 +10,6 @@ mod chat_template_tool_call_support; mod completion_usage_accounting; mod context_creation_limits; mod context_dimensions; -mod context_synchronization; mod debug_formatting; mod decode_abort_control; mod decode_batch_rejection; @@ -70,7 +69,6 @@ mod session_file_token_capacity; mod session_files; mod special_tokens; mod state_snapshots; -mod threadpool_detachment; mod token_attributes; mod token_candidates; mod tokenization; diff --git a/llama-cpp-bindings-tests/tests/threadpool_detachment.rs b/llama-cpp-bindings-tests/tests/threadpool_detachment.rs deleted file mode 100644 index 95f96020a..000000000 --- a/llama-cpp-bindings-tests/tests/threadpool_detachment.rs +++ /dev/null @@ -1,25 +0,0 @@ -use anyhow::Result; - -use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_test_harness::llama_fixture::LlamaFixture; -use llama_cpp_test_harness_macros::llama_test; - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn a_context_detaches_its_threadpool(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - context.detach_threadpool(); - - Ok(()) -} diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index db4dfc7c7..b46bd91a4 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -492,14 +492,6 @@ impl<'model> LlamaContext<'model> { } } - pub fn synchronize(&self) { - unsafe { llama_cpp_bindings_sys::llama_synchronize(self.context.as_ptr()) } - } - - pub fn detach_threadpool(&self) { - unsafe { llama_cpp_bindings_sys::llama_detach_threadpool(self.context.as_ptr()) } - } - /// Records the rows a llama.cpp helper wrote when it decoded on this context without one of /// this context's batches, such as the mtmd helper decoding media chunks. pub fn record_helper_decode(&mut self, logits_last: bool) { From ac93e1039e5ae5205e3c8abfe7f0482fecb11d3f Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 07:23:31 +0100 Subject: [PATCH 40/82] Classify with single peg parse flags and re-enable the enum range check --- .clang-tidy | 3 +- .../wrapper_chat_session.cpp | 22 ++----- .../tests/backend_initialization.rs | 18 ------ .../tests/generation_end_classification.rs | 57 +++++++++++++++++++ 4 files changed, 64 insertions(+), 36 deletions(-) delete mode 100644 llama-cpp-bindings-tests/tests/backend_initialization.rs diff --git a/.clang-tidy b/.clang-tidy index cf54a5bba..73eb4e2d7 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -12,7 +12,6 @@ Checks: > -mpi-*, -openmp-*, -zircon-*, - -bugprone-easily-swappable-parameters, - -clang-analyzer-optin.core.EnumCastOutOfRange + -bugprone-easily-swappable-parameters WarningsAsErrors: '*' FormatStyle: none diff --git a/llama-cpp-bindings-sys/wrapper_chat_session.cpp b/llama-cpp-bindings-sys/wrapper_chat_session.cpp index b0174880b..5373a27e1 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_session.cpp +++ b/llama-cpp-bindings-sys/wrapper_chat_session.cpp @@ -138,13 +138,8 @@ auto chat_input_of( return input; } -auto session_parser(const common_chat_session & session) -> const common_peg_arena & { - static const common_peg_arena content_only = - build_chat_peg_parser([](common_chat_peg_builder & builder) -> common_peg_parser { - return builder.content(builder.rest()) + builder.end(); - }); - - return session.parser().empty() ? content_only : session.parser(); +auto parse_flags_of(bool is_partial) -> common_peg_parse_flags { + return is_partial ? COMMON_PEG_PARSE_FLAG_LENIENT : COMMON_PEG_PARSE_FLAG_COMPLETE; } } // namespace @@ -236,7 +231,7 @@ extern "C" auto llama_rs_chat_session_create( chat_session->session.apply_sampling(chat_session->sampling); chat_session->preserved_tokens.assign( chat_session->sampling.preserved_tokens.begin(), chat_session->sampling.preserved_tokens.end()); - chat_session->trigger_rules = trigger_rules_of(session_parser(chat_session->session)); + chat_session->trigger_rules = trigger_rules_of(chat_session->session.parser()); *out_session = chat_session.release(); @@ -462,14 +457,9 @@ extern "C" auto llama_rs_chat_session_classify( common_chat_input parse_input = chat_input_of(session->generation_prompt, text, text_length, byte_tokens); const size_t generation_prompt_length = session->generation_prompt.size(); - common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_LENIENT; - if (!is_partial) { - flags |= COMMON_PEG_PARSE_FLAG_COMPLETE; - } - - common_peg_parse_context context(std::move(parse_input.text), std::move(parse_input.tokens), flags); - const common_peg_arena & parser = session_parser(session->session); - const common_peg_parse_result result = parser.parse(context); + common_peg_parse_context context( + std::move(parse_input.text), std::move(parse_input.tokens), parse_flags_of(is_partial)); + const common_peg_parse_result result = session->session.parser().parse(context); if (result.fail() && (!is_partial || result.end == 0)) { return LLAMA_RS_CHAT_SESSION_CLASSIFY_OUTPUT_DOES_NOT_MATCH_FORMAT; diff --git a/llama-cpp-bindings-tests/tests/backend_initialization.rs b/llama-cpp-bindings-tests/tests/backend_initialization.rs deleted file mode 100644 index bf85c0974..000000000 --- a/llama-cpp-bindings-tests/tests/backend_initialization.rs +++ /dev/null @@ -1,18 +0,0 @@ -use llama_cpp_test_harness::llama_fixture::LlamaFixture; -use llama_cpp_test_harness_macros::llama_test; - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - void_logs = true, -)] -fn a_model_loads_after_llama_cpp_logging_is_silenced(fixture: &LlamaFixture<'_>) { - assert!( - fixture.model.n_vocab() > 0, - "model must load successfully even when void_logs has been called before init" - ); -} diff --git a/llama-cpp-bindings-tests/tests/generation_end_classification.rs b/llama-cpp-bindings-tests/tests/generation_end_classification.rs index 99a84625f..9cf372db8 100644 --- a/llama-cpp-bindings-tests/tests/generation_end_classification.rs +++ b/llama-cpp-bindings-tests/tests/generation_end_classification.rs @@ -2,8 +2,11 @@ use anyhow::Result; use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::ingest_outcome::IngestOutcome; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_bindings::sampled_token_section::SampledTokenSection; use llama_cpp_bindings_tests::ingested_generation::ingested_generation; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; use llama_cpp_test_harness::llama_fixture::LlamaFixture; @@ -47,6 +50,60 @@ fn qwen35_classifier_ends_generation_without_emitting_the_end_of_generation_toke Ok(()) } +fn visible_text_in(outcomes: &[IngestOutcome], section: SampledTokenSection) -> String { + outcomes + .iter() + .filter(|outcome| outcome.sampled_token.section() == section) + .map(|outcome| outcome.piece.visible()) + .collect() +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn an_answer_stopped_inside_unclosed_reasoning_releases_its_unsettled_tail_as_content( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + const SETTLED_REASONING: &str = "Let me think"; + const UNSETTLED_TAIL: &str = " Date: Mon, 2 Nov 2026 07:23:31 +0100 Subject: [PATCH 41/82] Remove void_logs and let later send_logs_to_log calls replace the log options --- llama-cpp-bindings-tests/tests/main.rs | 1 - llama-cpp-bindings/src/send_logs_to_log.rs | 57 ++++++++++++++++--- .../src/attribute_accumulator.rs | 1 - llama-cpp-test-harness-macros/src/expand.rs | 6 +- llama-cpp-test-harness-macros/src/lib.rs | 2 +- .../src/parsed_args.rs | 50 +--------------- llama-cpp-test-harness/src/execution_plan.rs | 43 -------------- .../src/llama_test_registration.rs | 1 - .../src/run_to_conclusions.rs | 5 -- llama-cpp-test-harness/src/scheduled_phase.rs | 1 - .../tests/phase_conclusions.rs | 5 +- 11 files changed, 54 insertions(+), 118 deletions(-) diff --git a/llama-cpp-bindings-tests/tests/main.rs b/llama-cpp-bindings-tests/tests/main.rs index e1a6ab49d..58c7d577e 100644 --- a/llama-cpp-bindings-tests/tests/main.rs +++ b/llama-cpp-bindings-tests/tests/main.rs @@ -1,5 +1,4 @@ mod approximate_tok_env_eos_selection; -mod backend_initialization; mod batch_context_limits; mod chat_answer_parsing; mod chat_prompt_rendering; diff --git a/llama-cpp-bindings/src/send_logs_to_log.rs b/llama-cpp-bindings/src/send_logs_to_log.rs index 87f964670..285339685 100644 --- a/llama-cpp-bindings/src/send_logs_to_log.rs +++ b/llama-cpp-bindings/src/send_logs_to_log.rs @@ -30,7 +30,7 @@ use crate::log_record::LogRecord; struct LogSource { decoder: Mutex, target: &'static str, - options: LogOptions, + options: Mutex, } impl LogSource { @@ -38,9 +38,20 @@ impl LogSource { Self { decoder: Mutex::new(LogDecoder::new()), target, - options, + options: Mutex::new(options), } } + + fn options(&self) -> LogOptions { + self.options + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + + fn replace_options(&self, options: LogOptions) { + *self.options.lock().unwrap_or_else(PoisonError::into_inner) = options; + } } static LLAMA_SOURCE: OnceLock = OnceLock::new(); @@ -91,7 +102,7 @@ fn resolve_record(line: LogLine, demote_info_to_debug: bool) -> LogRecord { } fn dispatch_line(source: &LogSource, line: LogLine) { - let LogRecord { level, text } = resolve_record(line, source.options.demote_info_to_debug); + let LogRecord { level, text } = resolve_record(line, source.options().demote_info_to_debug); log!(target: source.target, level, "{text}"); } @@ -120,7 +131,7 @@ unsafe extern "C" fn logs_to_log( ) { let source: &LogSource = unsafe { &*data_ptr.cast::() }; - if source.options.disabled { + if source.options().disabled { return; } @@ -153,9 +164,15 @@ unsafe extern "C" fn logs_to_log( } pub fn send_logs_to_log(options: LogOptions) { - let llama_source: *const LogSource = - LLAMA_SOURCE.get_or_init(|| LogSource::new("llama.cpp", options.clone())); - let ggml_source: *const LogSource = GGML_SOURCE.get_or_init(|| LogSource::new("ggml", options)); + let llama_source = + LLAMA_SOURCE.get_or_init(|| LogSource::new("llama.cpp", LogOptions::default())); + let ggml_source = GGML_SOURCE.get_or_init(|| LogSource::new("ggml", LogOptions::default())); + + llama_source.replace_options(options.clone()); + ggml_source.replace_options(options); + + let llama_source: *const LogSource = llama_source; + let ggml_source: *const LogSource = ggml_source; unsafe { llama_cpp_bindings_sys::llama_log_set( @@ -486,6 +503,32 @@ mod tests { ); } + #[test] + #[serial] + fn sending_logs_again_applies_the_new_options() { + ensure_test_logger_installed(); + send_logs_to_log(LogOptions::default()); + send_logs_to_log(LogOptions::default().with_logs_enabled(false)); + + let mut ggml_callback: llama_cpp_bindings_sys::ggml_log_callback = None; + let mut ggml_user_data: *mut c_void = ptr::null_mut(); + + unsafe { + llama_cpp_bindings_sys::ggml_log_get(&raw mut ggml_callback, &raw mut ggml_user_data); + ggml_callback.unwrap()( + llama_cpp_bindings_sys::GGML_LOG_LEVEL_WARN, + c"silenced by the second call\n".as_ptr(), + ggml_user_data, + ); + } + send_logs_to_log(LogOptions::default()); + + assert_eq!( + records_with_text("ggml", "silenced by the second call"), + Vec::::new() + ); + } + #[test] #[serial] fn sending_logs_to_log_creates_the_llama_cpp_source_and_routes_ggml_logs() { diff --git a/llama-cpp-test-harness-macros/src/attribute_accumulator.rs b/llama-cpp-test-harness-macros/src/attribute_accumulator.rs index f1e1b7077..c5160ed2c 100644 --- a/llama-cpp-test-harness-macros/src/attribute_accumulator.rs +++ b/llama-cpp-test-harness-macros/src/attribute_accumulator.rs @@ -13,5 +13,4 @@ pub struct AttributeAccumulator { pub embeddings: Option, pub n_seq_max: Option, pub n_threads_batch: Option, - pub void_logs: Option, } diff --git a/llama-cpp-test-harness-macros/src/expand.rs b/llama-cpp-test-harness-macros/src/expand.rs index 66ae505e3..609c5303c 100644 --- a/llama-cpp-test-harness-macros/src/expand.rs +++ b/llama-cpp-test-harness-macros/src/expand.rs @@ -127,7 +127,6 @@ fn build_registration(args: &ParsedArgs, signature: &Signature) -> TokenStream { let physical_batch = args.context_params.n_ubatch; let embeddings_flag = args.context_params.embeddings; let sequence_max = args.context_params.n_seq_max; - let void_logs_flag = args.void_logs; let test_fn = build_test_fn_literal(signature); let threads_batch = args.context_params.n_threads_batch.map_or_else( || quote! { ::core::option::Option::None }, @@ -154,7 +153,6 @@ fn build_registration(args: &ParsedArgs, signature: &Signature) -> TokenStream { n_threads_batch: #threads_batch, embeddings: #embeddings_flag, }, - void_logs: #void_logs_flag, func: #test_fn, } } @@ -248,7 +246,6 @@ mod tests { n_threads_batch: ::core::option::Option::None, embeddings: false, }, - void_logs: false, func: ::llama_cpp_test_harness::llama_test_fn::LlamaTestFn::Fallible(my_test), } } @@ -324,8 +321,7 @@ mod tests { assert_eq!( expansion_error(quote! { totally_wrong = "x" }, well_formed_function()), "unknown field `totally_wrong`; expected one of: model_source, n_gpu_layers, load_mode, \ - n_ctx, n_batch, n_ubatch, mmproj_source, embeddings, n_seq_max, n_threads_batch, \ - void_logs" + n_ctx, n_batch, n_ubatch, mmproj_source, embeddings, n_seq_max, n_threads_batch" ); } diff --git a/llama-cpp-test-harness-macros/src/lib.rs b/llama-cpp-test-harness-macros/src/lib.rs index 7d2b4b6f6..5a4e0ac1d 100644 --- a/llama-cpp-test-harness-macros/src/lib.rs +++ b/llama-cpp-test-harness-macros/src/lib.rs @@ -43,7 +43,7 @@ mod tests { }; let message = "unknown field `totally_wrong`; expected one of: model_source, \ n_gpu_layers, load_mode, n_ctx, n_batch, n_ubatch, mmproj_source, \ - embeddings, n_seq_max, n_threads_batch, void_logs"; + embeddings, n_seq_max, n_threads_batch"; assert_eq!( dispatch(attribute, item).to_string(), diff --git a/llama-cpp-test-harness-macros/src/parsed_args.rs b/llama-cpp-test-harness-macros/src/parsed_args.rs index bb2306c7a..56cb6184e 100644 --- a/llama-cpp-test-harness-macros/src/parsed_args.rs +++ b/llama-cpp-test-harness-macros/src/parsed_args.rs @@ -33,7 +33,6 @@ const OPTIONAL_FIELDS: &[&str] = &[ "embeddings", "n_seq_max", "n_threads_batch", - "void_logs", ]; fn literal_from_expression(expression: &Expr) -> syn::Result<&Lit> { @@ -186,12 +185,6 @@ fn dispatch_field( "n_threads_batch" => { accumulator.n_threads_batch = Some(require_i32_value(value, "n_threads_batch")?); } - "void_logs" => { - accumulator.void_logs = Some(require_bool_lit( - literal_from_expression(value)?, - "void_logs", - )?); - } "repo" | "file" | "mmproj_file" => { return Err(syn::Error::new_spanned( identifier, @@ -220,7 +213,6 @@ pub struct ParsedArgs { pub mmproj_source: Option, pub model_load_params: ParsedModelLoadParams, pub context_params: ParsedContextParams, - pub void_logs: bool, } impl Parse for ParsedArgs { @@ -266,7 +258,6 @@ impl Parse for ParsedArgs { n_threads_batch: accumulator.n_threads_batch, embeddings: accumulator.embeddings.unwrap_or(false), }, - void_logs: accumulator.void_logs.unwrap_or(false), }) } } @@ -413,8 +404,7 @@ mod tests { assert_eq!( message, "unknown field `surprise`; expected one of: model_source, n_gpu_layers, load_mode, \ - n_ctx, n_batch, n_ubatch, mmproj_source, embeddings, n_seq_max, n_threads_batch, \ - void_logs" + n_ctx, n_batch, n_ubatch, mmproj_source, embeddings, n_seq_max, n_threads_batch" ); } @@ -759,21 +749,6 @@ mod tests { ); } - #[test] - fn optional_void_logs_defaults_to_false_when_absent() { - let parsed = parse(ALL_REQUIRED).expect("required-only must parse"); - - assert!(!parsed.void_logs); - } - - #[test] - fn optional_void_logs_true_is_parsed() { - let source = format!("{ALL_REQUIRED}, void_logs = true"); - let parsed = parse(&source).expect("void_logs = true must parse"); - - assert!(parsed.void_logs); - } - fn override_field(field: &str, replacement: &str) -> String { let parts: [(&str, &str); 6] = [ ("model_source", "HuggingFace(\"foo\", \"bar.gguf\")"), @@ -863,29 +838,6 @@ mod tests { assert_eq!(message, "expected a literal (string, integer, or bool)"); } - #[test] - fn optional_void_logs_rejects_non_literal_value() { - let source = append_field("void_logs", "some_const"); - let message = parse(&source) - .expect_err("void_logs non-literal must fail") - .to_string(); - - assert_eq!(message, "expected a literal (string, integer, or bool)"); - } - - #[test] - fn optional_void_logs_rejects_wrong_literal_kind() { - let source = append_field("void_logs", "1"); - let message = parse(&source) - .expect_err("void_logs wrong-kind must fail") - .to_string(); - - assert_eq!( - message, - "field `void_logs` expects a bool literal (`true` or `false`)" - ); - } - #[test] fn optional_mmproj_source_rejects_unknown_variant() { let source = append_field("mmproj_source", "Mystery(\"a\", \"b\")"); diff --git a/llama-cpp-test-harness/src/execution_plan.rs b/llama-cpp-test-harness/src/execution_plan.rs index 4ae45f60e..08c2467b7 100644 --- a/llama-cpp-test-harness/src/execution_plan.rs +++ b/llama-cpp-test-harness/src/execution_plan.rs @@ -44,13 +44,6 @@ impl ExecutionPlan { Self::from_registrations(®istrations) } - #[must_use] - pub fn requests_void_logs(&self) -> bool { - self.phases - .iter() - .any(|phase| phase.registrations.iter().any(|reg| reg.void_logs)) - } - #[must_use] pub fn run(&self, backend: &Arc, arguments: &Arguments) -> Vec { let total = self.phases.len(); @@ -117,7 +110,6 @@ mod tests { }, }, context_params: TRIVIAL_CONTEXT_PARAMS, - void_logs: false, func: LlamaTestFn::Infallible(no_op), }; static REG_BETA_B: LlamaTestRegistration = LlamaTestRegistration { @@ -134,7 +126,6 @@ mod tests { }, }, context_params: TRIVIAL_CONTEXT_PARAMS, - void_logs: false, func: LlamaTestFn::Infallible(no_op), }; static REG_ALPHA_Z: LlamaTestRegistration = LlamaTestRegistration { @@ -151,7 +142,6 @@ mod tests { }, }, context_params: TRIVIAL_CONTEXT_PARAMS, - void_logs: false, func: LlamaTestFn::Infallible(no_op), }; static REG_BETA_DIFFERENT_CONTEXT: LlamaTestRegistration = LlamaTestRegistration { @@ -168,25 +158,6 @@ mod tests { }, }, context_params: ALTERNATE_CONTEXT_PARAMS, - void_logs: false, - func: LlamaTestFn::Infallible(no_op), - }; - - static REG_VOID_LOGS: LlamaTestRegistration = LlamaTestRegistration { - name: "void-logs-trial", - key: LoadKey { - model_source: GgufSource::HuggingFace { - repo: "beta", - file: "f", - }, - mmproj_source: None, - model_load_params: ModelLoadParams { - n_gpu_layers: 0, - load_mode: LlamaLoadMode::Mmap, - }, - }, - context_params: TRIVIAL_CONTEXT_PARAMS, - void_logs: true, func: LlamaTestFn::Infallible(no_op), }; @@ -235,20 +206,6 @@ mod tests { assert_eq!(plan.phases[0].registrations[1].name, "bravo"); } - #[test] - fn requests_void_logs_false_when_no_registration_opts_in() { - let plan = ExecutionPlan::from_registrations(&[®_BETA_A, ®_ALPHA_Z]); - - assert!(!plan.requests_void_logs()); - } - - #[test] - fn requests_void_logs_true_when_any_registration_opts_in() { - let plan = ExecutionPlan::from_registrations(&[®_BETA_A, ®_VOID_LOGS]); - - assert!(plan.requests_void_logs()); - } - #[test] fn registrations_sharing_a_load_key_but_differing_context_params_stay_in_one_phase() { let plan = ExecutionPlan::from_registrations(&[®_BETA_A, ®_BETA_DIFFERENT_CONTEXT]); diff --git a/llama-cpp-test-harness/src/llama_test_registration.rs b/llama-cpp-test-harness/src/llama_test_registration.rs index 80f24f5da..1d6ce5e7b 100644 --- a/llama-cpp-test-harness/src/llama_test_registration.rs +++ b/llama-cpp-test-harness/src/llama_test_registration.rs @@ -8,7 +8,6 @@ pub struct LlamaTestRegistration { pub name: &'static str, pub key: LoadKey, pub context_params: ContextParams, - pub void_logs: bool, pub func: LlamaTestFn, } diff --git a/llama-cpp-test-harness/src/run_to_conclusions.rs b/llama-cpp-test-harness/src/run_to_conclusions.rs index 97118959f..86e1f24a3 100644 --- a/llama-cpp-test-harness/src/run_to_conclusions.rs +++ b/llama-cpp-test-harness/src/run_to_conclusions.rs @@ -4,8 +4,6 @@ use libtest_mimic::Arguments; use libtest_mimic::Conclusion; use llama_cpp_bindings::llama_backend::LlamaBackend; -use llama_cpp_bindings::log_options::LogOptions; -use llama_cpp_bindings::send_logs_to_log::send_logs_to_log; use crate::execution_plan::ExecutionPlan; use crate::harness_run_error::HarnessRunError; @@ -20,9 +18,6 @@ pub fn run_to_conclusions(arguments: Arguments) -> Result, Harne let arguments = parse_harness_arguments(arguments)?; let backend = Arc::new(LlamaBackend::init()?); let plan = ExecutionPlan::from_inventory(); - if plan.requests_void_logs() { - send_logs_to_log(LogOptions::default().with_logs_enabled(false)); - } Ok(plan.run(&backend, &arguments)) } diff --git a/llama-cpp-test-harness/src/scheduled_phase.rs b/llama-cpp-test-harness/src/scheduled_phase.rs index 9247fdaa2..9644061fc 100644 --- a/llama-cpp-test-harness/src/scheduled_phase.rs +++ b/llama-cpp-test-harness/src/scheduled_phase.rs @@ -125,7 +125,6 @@ mod tests { n_threads_batch: None, embeddings: false, }, - void_logs: false, func: LlamaTestFn::Infallible(no_op), }; diff --git a/llama-cpp-test-harness/tests/phase_conclusions.rs b/llama-cpp-test-harness/tests/phase_conclusions.rs index a2fcffa02..e1fadfed6 100644 --- a/llama-cpp-test-harness/tests/phase_conclusions.rs +++ b/llama-cpp-test-harness/tests/phase_conclusions.rs @@ -16,11 +16,8 @@ use llama_cpp_test_harness_macros::llama_test; n_ctx = 512, n_batch = 128, n_ubatch = 64, - void_logs = true, )] -fn phase_a_passing_trial_that_requests_void_logs(fixture: &LlamaFixture<'_>) { - let formatted = format!("{:?}", fixture.model); - assert!(formatted.contains("LlamaModel")); +const fn phase_a_passing_trial_that_only_needs_its_model_to_load(fixture: &LlamaFixture<'_>) { no_op(fixture); } From b6054c6c4b0a8b2587a006a6b27e14016aaa204f Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 07:29:29 +0100 Subject: [PATCH 42/82] Replace duck-typed tool calls with llama.cpp chat template overrides --- Cargo.lock | 13 +- Cargo.toml | 1 - .../wrapper_chat_templates.cpp | 29 +- .../wrapper_chat_templates.h | 6 +- .../wrapper_chat_templates_handle.h | 1 - .../src/chat_answer_generation.rs | 4 +- .../src/closed_reasoning_chat_session.rs | 3 + .../src/user_turn_chat_session.rs | 1 + .../src/weather_chat_session.rs | 3 + .../tests/chat_prompt_rendering.rs | 1 + .../tests/chat_session_input_validation.rs | 2 + .../tests/chat_session_llama_cpp_failures.rs | 7 +- .../tests/chat_template_override.rs | 151 +++++ .../tests/chat_template_tool_call_support.rs | 18 - .../tests/completion_usage_accounting.rs | 5 +- .../tests/duck_typed_tool_calls.rs | 116 ---- .../tests/generation_end_classification.rs | 63 +-- .../tests/grammar_sampler.rs | 4 +- .../tests/image_question_answering.rs | 4 +- llama-cpp-bindings-tests/tests/main.rs | 3 +- .../tests/multimodal_audio.rs | 5 +- .../tests/multimodal_fusion.rs | 5 +- .../multimodal_reasoning_classification.rs | 11 +- .../tests/reasoning_classification.rs | 1 + .../tests/sampled_token_ingestion.rs | 18 +- .../tests/tool_call_classification.rs | 4 +- .../tests/tool_call_parsing.rs | 1 + .../src/bracketed_json_shape.rs | 4 - .../src/json_object_shape.rs | 5 - .../src/key_value_xml_tags_shape.rs | 7 - llama-cpp-bindings-types/src/lib.rs | 8 - .../src/paired_quote_shape.rs | 7 - .../src/tool_call_args_shape.rs | 14 - .../src/tool_call_markers.rs | 8 - .../src/tool_call_value_quote.rs | 5 - .../src/xml_tags_shape.rs | 7 - llama-cpp-bindings/Cargo.toml | 2 - llama-cpp-bindings/src/bare_json_tool_call.rs | 10 - .../src/bare_json_tool_call_field.rs | 5 - .../src/bare_json_tool_call_probe.rs | 233 -------- .../src/bare_json_tool_call_probe_state.rs | 11 - .../src/bare_json_tool_calls.rs | 5 - .../src/chat_message_parse_outcome.rs | 60 -- llama-cpp-bindings/src/chat_session.rs | 81 +-- .../src/chat_session_attribution.rs | 2 - .../src/chat_template_tool_calls.rs | 6 - .../gemma4_call_block.rs | 23 - .../glm47_key_value_tags.rs | 21 - .../known_marker_candidates.rs | 132 ----- .../mistral3_arrow_args.rs | 18 - .../qwen3_json_inside_tool_call.rs | 19 - .../chat_template_tool_calls/qwen_xml_tags.rs | 21 - llama-cpp-bindings/src/chat_templates.rs | 64 ++- llama-cpp-bindings/src/content_probe.rs | 23 - llama-cpp-bindings/src/error.rs | 6 - .../src/error/bracketed_args_failure.rs | 7 - .../src/error/chat_templates_error.rs | 4 + .../src/error/json_object_failure.rs | 5 - .../src/error/key_value_xml_tags_failure.rs | 30 - .../src/error/paired_quote_failure.rs | 22 - .../src/error/tool_call_format_failure.rs | 19 - .../src/error/xml_function_tags_failure.rs | 20 - llama-cpp-bindings/src/json_probe_outcome.rs | 6 - llama-cpp-bindings/src/lib.rs | 11 - .../src/sampled_token_classifier.rs | 30 +- .../src/streaming_json_probe.rs | 379 ------------- .../src/streaming_json_probe_state.rs | 157 ------ llama-cpp-bindings/src/tool_call_format.rs | 236 -------- .../src/tool_call_format/bracketed_args.rs | 266 --------- .../consume_optional_prefix.rs | 4 - .../src/tool_call_format/json_object.rs | 197 ------- .../tool_call_format/key_value_xml_tags.rs | 339 ------------ .../src/tool_call_format/paired_quote_args.rs | 515 ------------------ .../src/tool_call_format/parse_step.rs | 10 - .../src/tool_call_format/parsed_prefix.rs | 5 - .../tool_call_format/scalar_value_to_json.rs | 5 - .../src/tool_call_format/separator_split.rs | 14 - .../tool_call_format_outcome.rs | 10 - .../src/tool_call_format/xml_function_tags.rs | 363 ------------ 79 files changed, 280 insertions(+), 3661 deletions(-) create mode 100644 llama-cpp-bindings-tests/tests/chat_template_override.rs delete mode 100644 llama-cpp-bindings-tests/tests/chat_template_tool_call_support.rs delete mode 100644 llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs delete mode 100644 llama-cpp-bindings-types/src/bracketed_json_shape.rs delete mode 100644 llama-cpp-bindings-types/src/json_object_shape.rs delete mode 100644 llama-cpp-bindings-types/src/key_value_xml_tags_shape.rs delete mode 100644 llama-cpp-bindings-types/src/paired_quote_shape.rs delete mode 100644 llama-cpp-bindings-types/src/tool_call_args_shape.rs delete mode 100644 llama-cpp-bindings-types/src/tool_call_markers.rs delete mode 100644 llama-cpp-bindings-types/src/tool_call_value_quote.rs delete mode 100644 llama-cpp-bindings-types/src/xml_tags_shape.rs delete mode 100644 llama-cpp-bindings/src/bare_json_tool_call.rs delete mode 100644 llama-cpp-bindings/src/bare_json_tool_call_field.rs delete mode 100644 llama-cpp-bindings/src/bare_json_tool_call_probe.rs delete mode 100644 llama-cpp-bindings/src/bare_json_tool_call_probe_state.rs delete mode 100644 llama-cpp-bindings/src/bare_json_tool_calls.rs delete mode 100644 llama-cpp-bindings/src/chat_template_tool_calls.rs delete mode 100644 llama-cpp-bindings/src/chat_template_tool_calls/gemma4_call_block.rs delete mode 100644 llama-cpp-bindings/src/chat_template_tool_calls/glm47_key_value_tags.rs delete mode 100644 llama-cpp-bindings/src/chat_template_tool_calls/known_marker_candidates.rs delete mode 100644 llama-cpp-bindings/src/chat_template_tool_calls/mistral3_arrow_args.rs delete mode 100644 llama-cpp-bindings/src/chat_template_tool_calls/qwen3_json_inside_tool_call.rs delete mode 100644 llama-cpp-bindings/src/chat_template_tool_calls/qwen_xml_tags.rs delete mode 100644 llama-cpp-bindings/src/content_probe.rs delete mode 100644 llama-cpp-bindings/src/error/bracketed_args_failure.rs delete mode 100644 llama-cpp-bindings/src/error/json_object_failure.rs delete mode 100644 llama-cpp-bindings/src/error/key_value_xml_tags_failure.rs delete mode 100644 llama-cpp-bindings/src/error/paired_quote_failure.rs delete mode 100644 llama-cpp-bindings/src/error/tool_call_format_failure.rs delete mode 100644 llama-cpp-bindings/src/error/xml_function_tags_failure.rs delete mode 100644 llama-cpp-bindings/src/json_probe_outcome.rs delete mode 100644 llama-cpp-bindings/src/streaming_json_probe.rs delete mode 100644 llama-cpp-bindings/src/streaming_json_probe_state.rs delete mode 100644 llama-cpp-bindings/src/tool_call_format.rs delete mode 100644 llama-cpp-bindings/src/tool_call_format/bracketed_args.rs delete mode 100644 llama-cpp-bindings/src/tool_call_format/consume_optional_prefix.rs delete mode 100644 llama-cpp-bindings/src/tool_call_format/json_object.rs delete mode 100644 llama-cpp-bindings/src/tool_call_format/key_value_xml_tags.rs delete mode 100644 llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs delete mode 100644 llama-cpp-bindings/src/tool_call_format/parse_step.rs delete mode 100644 llama-cpp-bindings/src/tool_call_format/parsed_prefix.rs delete mode 100644 llama-cpp-bindings/src/tool_call_format/scalar_value_to_json.rs delete mode 100644 llama-cpp-bindings/src/tool_call_format/separator_split.rs delete mode 100644 llama-cpp-bindings/src/tool_call_format/tool_call_format_outcome.rs delete mode 100644 llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs diff --git a/Cargo.lock b/Cargo.lock index 9a01275ca..b1a70dfbd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -190,7 +190,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom 7.1.3", + "nom", ] [[package]] @@ -1178,9 +1178,7 @@ dependencies = [ "llama-cpp-wrapper-error-fixture", "llguidance", "log", - "nom 8.0.0", "once_cell", - "serde", "serde_json", "serial_test", "thiserror", @@ -1407,15 +1405,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "nom" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" -dependencies = [ - "memchr", -] - [[package]] name = "num-conv" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 5d9af5ca5..7da283e83 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,7 +49,6 @@ llama-cpp-wrapper-error-fixture = { path = "llama-cpp-wrapper-error-fixture" } llama-cpp-wrapper-sources = { path = "llama-cpp-wrapper-sources", version = "=0.26.0" } llguidance = "=1.7.0" log = "=0.4.29" -nom = "=8.0.0" once_cell = "1.21" proc-macro2 = "=1.0.106" quote = "=1.0.45" diff --git a/llama-cpp-bindings-sys/wrapper_chat_templates.cpp b/llama-cpp-bindings-sys/wrapper_chat_templates.cpp index 64809ccad..4aa9f6982 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_templates.cpp +++ b/llama-cpp-bindings-sys/wrapper_chat_templates.cpp @@ -3,26 +3,14 @@ #include "wrapper_report_current_exception.h" #include -#include -#include -#include -#include #include #include #include -namespace { -auto template_supports_tool_calls(const std::string & source) -> bool { - jinja::lexer lexer; - jinja::program program = jinja::parse_from_tokens(lexer.tokenize(source)); - - return jinja::caps_get(program).supports_tool_calls; -} -} // namespace - extern "C" auto llama_rs_chat_templates_init( const struct llama_model * model, + const char * template_override, llama_rs_chat_templates_handle * out_templates, char ** out_error) -> llama_rs_chat_templates_init_status { if (out_templates != nullptr) { @@ -34,23 +22,22 @@ extern "C" auto llama_rs_chat_templates_init( if (model == nullptr) { return LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_MODEL_ARG; } + if (template_override == nullptr) { + return LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_TEMPLATE_OVERRIDE_ARG; + } if (out_templates == nullptr) { return LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_TEMPLATES_ARG; } if (out_error == nullptr) { return LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_ERROR_ARG; } - if (llama_model_chat_template(model, nullptr) == nullptr) { + if (*template_override == '\0' && llama_model_chat_template(model, nullptr) == nullptr) { return LLAMA_RS_CHAT_TEMPLATES_INIT_MODEL_HAS_NO_CHAT_TEMPLATE; } try { auto templates = std::make_unique(); - templates->templates = common_chat_templates_init(model, std::string()); - - const std::string tool_use_source = common_chat_templates_source(templates->templates.get(), "tool_use"); - templates->supports_tool_calls = template_supports_tool_calls( - tool_use_source.empty() ? common_chat_templates_source(templates->templates.get()) : tool_use_source); + templates->templates = common_chat_templates_init(model, std::string(template_override)); *out_templates = templates.release(); @@ -67,7 +54,3 @@ extern "C" auto llama_rs_chat_templates_init( extern "C" auto llama_rs_chat_templates_free(llama_rs_chat_templates_handle templates) -> void { std::unique_ptr(templates).reset(); } - -extern "C" auto llama_rs_chat_templates_supports_tool_calls(llama_rs_chat_templates_handle templates) -> bool { - return templates->supports_tool_calls; -} diff --git a/llama-cpp-bindings-sys/wrapper_chat_templates.h b/llama-cpp-bindings-sys/wrapper_chat_templates.h index bdaa5ca88..1c343dc87 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_templates.h +++ b/llama-cpp-bindings-sys/wrapper_chat_templates.h @@ -2,8 +2,6 @@ #include -#include - #ifdef __cplusplus extern "C" { #endif @@ -14,6 +12,7 @@ typedef struct llama_rs_chat_templates * llama_rs_chat_templates_handle; typedef enum llama_rs_chat_templates_init_status { LLAMA_RS_CHAT_TEMPLATES_INIT_OK = 0, LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_MODEL_ARG, + LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_TEMPLATE_OVERRIDE_ARG, LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_TEMPLATES_ARG, LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_ERROR_ARG, LLAMA_RS_CHAT_TEMPLATES_INIT_MODEL_HAS_NO_CHAT_TEMPLATE, @@ -24,13 +23,12 @@ typedef enum llama_rs_chat_templates_init_status { llama_rs_chat_templates_init_status llama_rs_chat_templates_init( const struct llama_model * model, + const char * template_override, llama_rs_chat_templates_handle * out_templates, char ** out_error); void llama_rs_chat_templates_free(llama_rs_chat_templates_handle templates); -bool llama_rs_chat_templates_supports_tool_calls(llama_rs_chat_templates_handle templates); - #ifdef __cplusplus } #endif diff --git a/llama-cpp-bindings-sys/wrapper_chat_templates_handle.h b/llama-cpp-bindings-sys/wrapper_chat_templates_handle.h index 69d8bbb12..8b7723ff7 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_templates_handle.h +++ b/llama-cpp-bindings-sys/wrapper_chat_templates_handle.h @@ -4,5 +4,4 @@ struct llama_rs_chat_templates { common_chat_templates_ptr templates; - bool supports_tool_calls = false; }; diff --git a/llama-cpp-bindings-tests/src/chat_answer_generation.rs b/llama-cpp-bindings-tests/src/chat_answer_generation.rs index 6055dff07..d5402ca0f 100644 --- a/llama-cpp-bindings-tests/src/chat_answer_generation.rs +++ b/llama-cpp-bindings-tests/src/chat_answer_generation.rs @@ -1,4 +1,3 @@ -use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; @@ -36,8 +35,7 @@ impl ChatAnswerGeneration<'_, '_> { let prompt_tokens = session .tokenize_prompt() .expect("the session's prompt must tokenize"); - let mut classifier = - SampledTokenClassifier::for_chat_session(session, BareJsonToolCalls::Detect); + let mut classifier = SampledTokenClassifier::for_chat_session(session); let mut batch = LlamaBatch::new(ContextLimits::of(&context)); classifier diff --git a/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs b/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs index d703c5540..941a59811 100644 --- a/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs +++ b/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs @@ -25,6 +25,9 @@ pub fn closed_reasoning_chat_session<'model>( ) -> ChatSession<'model> { ChatSession::new( model, + model + .chat_templates() + .expect("the model must have chat templates"), &[ LlamaChatMessage::new("user".to_owned(), question.to_owned()) .expect("the question must hold no NUL byte"), diff --git a/llama-cpp-bindings-tests/src/user_turn_chat_session.rs b/llama-cpp-bindings-tests/src/user_turn_chat_session.rs index 14d656819..72805e2c4 100644 --- a/llama-cpp-bindings-tests/src/user_turn_chat_session.rs +++ b/llama-cpp-bindings-tests/src/user_turn_chat_session.rs @@ -20,6 +20,7 @@ pub fn user_turn_chat_session<'model>( ) -> Result> { Ok(ChatSession::new( model, + model.chat_templates()?, &[LlamaChatMessage::new( "user".to_owned(), content.to_owned(), diff --git a/llama-cpp-bindings-tests/src/weather_chat_session.rs b/llama-cpp-bindings-tests/src/weather_chat_session.rs index 5373ffdd1..565fba001 100644 --- a/llama-cpp-bindings-tests/src/weather_chat_session.rs +++ b/llama-cpp-bindings-tests/src/weather_chat_session.rs @@ -17,6 +17,9 @@ use crate::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; pub fn weather_chat_session(model: &LlamaModel, tool_choice: ChatToolChoice) -> ChatSession<'_> { ChatSession::new( model, + model + .chat_templates() + .expect("the model must have chat templates"), &[LlamaChatMessage::new( "user".to_owned(), "What is the weather in Paris?".to_owned(), diff --git a/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs b/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs index 8c6f31169..1b35b4256 100644 --- a/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs +++ b/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs @@ -148,6 +148,7 @@ fn qwen35_omitted_assistant_turn_renders_only_the_messages( ) -> Result<()> { let session = ChatSession::new( fixture.model, + fixture.model.chat_templates()?, &[LlamaChatMessage::new( "user".to_owned(), "hello".to_owned(), diff --git a/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs index fc45fae32..d4810e82c 100644 --- a/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs +++ b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs @@ -103,6 +103,7 @@ fn chat_session_refuses_a_message_that_is_not_utf8(fixture: &LlamaFixture<'_>) - assert_eq!( ChatSession::new( fixture.model, + fixture.model.chat_templates()?, &[message], "[]", &ChatSessionOptions { @@ -135,6 +136,7 @@ fn tokenizing_a_prompt_whose_reasoning_prefill_holds_a_nul_character_is_refused( ) -> Result<()> { let session = ChatSession::new( fixture.model, + fixture.model.chat_templates()?, &[LlamaChatMessage::new( "user".to_owned(), "hello".to_owned(), diff --git a/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs b/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs index 40bff51ad..1fd2693eb 100644 --- a/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs +++ b/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs @@ -232,18 +232,15 @@ fn the_grammar_sampler_reports_a_lazy_grammar_llama_cpp_cannot_load(fixture: &Ll n_batch = 128, n_ubatch = 64, )] -fn parsing_a_duck_typed_tool_call_reports_a_parser_llama_cpp_fails(fixture: &LlamaFixture<'_>) { +fn parsing_an_answer_reports_a_parser_llama_cpp_fails(fixture: &LlamaFixture<'_>) { let session = weather_chat_session(fixture.model, ChatToolChoice::Auto); let _fault = ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_PARSE_THROWS); - assert!(!session.supports_tool_calls()); assert_eq!( session .parse( - &ChatOutput::from( - "Checking.\n\n\nParis\n\n\n" - ), + &ChatOutput::from("Checking."), ChatOutputCompleteness::Finished, ) .err(), diff --git a/llama-cpp-bindings-tests/tests/chat_template_override.rs b/llama-cpp-bindings-tests/tests/chat_template_override.rs new file mode 100644 index 000000000..5b1b71235 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/chat_template_override.rs @@ -0,0 +1,151 @@ +use std::fs; +use std::path::PathBuf; + +use anyhow::Result; + +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_templates::ChatTemplates; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; +use llama_cpp_bindings::error::chat_templates_error::ChatTemplatesError; +use llama_cpp_bindings::model::LlamaModel; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; +use llama_cpp_bindings::model::llama_chat_template::LlamaChatTemplate; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_bindings_tests::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; +use llama_cpp_bindings_tests::ingested_generation::ingested_generation; +use llama_cpp_bindings_tests::paris_weather_tool_call::paris_weather_tool_call; +use llama_cpp_bindings_tests::recognized_answer::recognized_answer; +use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +const REASONING: &str = "The user wants the weather in Paris."; +const R1_WEATHER_TOOL_CALL: &str = "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather\n```json\n{\"location\": \"Paris\"}```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; +const END_OF_GENERATION_TOKENS: u64 = 1; + +fn llama_cpp_deepseek_r1_template() -> Result { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../llama-cpp-bindings-sys/llama.cpp/models/templates/llama-cpp-deepseek-r1.jinja"); + + Ok(LlamaChatTemplate::new(&fs::read_to_string(path)?)?) +} + +fn weather_session<'model>( + model: &'model LlamaModel, + templates: &ChatTemplates, +) -> Result> { + Ok(ChatSession::new( + model, + templates, + &[LlamaChatMessage::new( + "user".to_owned(), + "What is the weather in Paris?".to_owned(), + )?], + GET_WEATHER_TOOLS_JSON, + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::New, + enable_thinking: true, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + )?) +} + +fn token_count(model: &LlamaModel, text: &str) -> Result { + Ok(u64::try_from( + model + .str_to_token(text, AddBos::Never, ParseSpecialTokens::Always)? + .len(), + )?) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 1024, + n_batch = 128, + n_ubatch = 64, +)] +fn deepseek_r1_served_with_the_llama_cpp_template_parses_its_tool_call( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let templates = + ChatTemplates::with_override(fixture.model, &llama_cpp_deepseek_r1_template()?)?; + let session = weather_session(fixture.model, &templates)?; + + assert_eq!( + recognized_answer( + &session, + &format!("{REASONING}{R1_WEATHER_TOOL_CALL}"), + ChatOutputCompleteness::Finished + ), + ParsedChatMessage::new( + String::new(), + "\nThe user wants the weather in Paris.".to_owned(), + vec![paris_weather_tool_call()] + ) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 1024, + n_batch = 128, + n_ubatch = 64, +)] +fn deepseek_r1_served_with_the_llama_cpp_template_streams_its_tool_call_tokens_as_a_tool_call( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let templates = + ChatTemplates::with_override(fixture.model, &llama_cpp_deepseek_r1_template()?)?; + let session = weather_session(fixture.model, &templates)?; + let mut classifier = SampledTokenClassifier::for_chat_session(&session); + + ingested_generation( + &mut classifier, + fixture.model, + &format!("{REASONING}{R1_WEATHER_TOOL_CALL}"), + ); + + assert_eq!( + [ + classifier.usage().reasoning_tokens, + classifier.usage().content_tokens, + classifier.usage().tool_call_tokens, + ], + [ + token_count(fixture.model, REASONING)?, + 0, + token_count(fixture.model, R1_WEATHER_TOOL_CALL)? + END_OF_GENERATION_TOKENS, + ] + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 1024, + n_batch = 128, + n_ubatch = 64, +)] +fn an_empty_template_override_is_refused(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_eq!( + ChatTemplates::with_override(fixture.model, &LlamaChatTemplate::new("")?).err(), + Some(ChatTemplatesError::EmptyTemplateOverride) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/chat_template_tool_call_support.rs b/llama-cpp-bindings-tests/tests/chat_template_tool_call_support.rs deleted file mode 100644 index 9b90a767e..000000000 --- a/llama-cpp-bindings-tests/tests/chat_template_tool_call_support.rs +++ /dev/null @@ -1,18 +0,0 @@ -use anyhow::Result; - -use llama_cpp_test_harness::llama_fixture::LlamaFixture; -use llama_cpp_test_harness_macros::llama_test; - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128 -)] -fn embedding_model_chat_template_supports_tool_calls(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(fixture.model.chat_templates()?.supports_tool_calls()); - - Ok(()) -} diff --git a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs index 62eeea7cb..f104faaba 100644 --- a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs +++ b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs @@ -1,7 +1,6 @@ use anyhow::Context as _; use anyhow::Result; -use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings::chat_session_options::ChatSessionOptions; @@ -184,6 +183,7 @@ fn chat_completion_usage_matches_the_classified_tokens(fixture: &LlamaFixture<'_ )?]; let session = ChatSession::new( model, + model.chat_templates()?, &messages, "[]", &ChatSessionOptions { @@ -194,8 +194,7 @@ fn chat_completion_usage_matches_the_classified_tokens(fixture: &LlamaFixture<'_ }, )?; - let mut classifier = - SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let mut classifier = SampledTokenClassifier::for_chat_session(&session); let tokens = session.tokenize_prompt()?; let prompt_token_count = u64::try_from(tokens.len())?; diff --git a/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs b/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs deleted file mode 100644 index 53deb4dc3..000000000 --- a/llama-cpp-bindings-tests/tests/duck_typed_tool_calls.rs +++ /dev/null @@ -1,116 +0,0 @@ -use anyhow::Result; - -use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; -use llama_cpp_bindings_tests::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; -use llama_cpp_bindings_tests::paris_weather_tool_call::paris_weather_tool_call; -use llama_cpp_bindings_tests::recognized_answer::recognized_answer; -use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; -use llama_cpp_test_harness::llama_fixture::LlamaFixture; -use llama_cpp_test_harness_macros::llama_test; - -fn assert_duck_types_paris_weather_call(fixture: &LlamaFixture<'_>, answer: &str) -> Result<()> { - let session = user_turn_chat_session( - fixture.model, - "What is the weather in Paris?", - GET_WEATHER_TOOLS_JSON, - false, - )?; - - assert!(!session.supports_tool_calls()); - assert_eq!( - recognized_answer(&session, answer, ChatOutputCompleteness::Finished).tool_calls, - [paris_weather_tool_call()] - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn deepseek_r1_8b_duck_types_gemma_paired_quote(fixture: &LlamaFixture<'_>) -> Result<()> { - assert_duck_types_paris_weather_call( - fixture, - "<|tool_call>call:get_weather{location:<|\"|>Paris<|\"|>}", - ) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn deepseek_r1_8b_duck_types_glm_key_value_tags(fixture: &LlamaFixture<'_>) -> Result<()> { - assert_duck_types_paris_weather_call( - fixture, - "get_weatherlocationParis", - ) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn deepseek_r1_8b_duck_types_mistral_bracketed_json(fixture: &LlamaFixture<'_>) -> Result<()> { - assert_duck_types_paris_weather_call( - fixture, - r#"[TOOL_CALLS]get_weather[ARGS]{"location":"Paris"}"#, - ) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn deepseek_r1_8b_duck_types_qwen_xml(fixture: &LlamaFixture<'_>) -> Result<()> { - assert_duck_types_paris_weather_call( - fixture, - "\n\n\nParis\n\n\n", - ) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn deepseek_r1_8b_duck_types_qwen_json_inside_tool_call(fixture: &LlamaFixture<'_>) -> Result<()> { - assert_duck_types_paris_weather_call( - fixture, - "\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Paris\"}}\n", - ) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn deepseek_r1_8b_duck_types_a_bare_json_tool_call(fixture: &LlamaFixture<'_>) -> Result<()> { - assert_duck_types_paris_weather_call( - fixture, - "{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Paris\"}}", - ) -} diff --git a/llama-cpp-bindings-tests/tests/generation_end_classification.rs b/llama-cpp-bindings-tests/tests/generation_end_classification.rs index 9cf372db8..b32192d0d 100644 --- a/llama-cpp-bindings-tests/tests/generation_end_classification.rs +++ b/llama-cpp-bindings-tests/tests/generation_end_classification.rs @@ -1,10 +1,8 @@ use anyhow::Result; -use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::ingest_outcome::IngestOutcome; use llama_cpp_bindings::model::add_bos::AddBos; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; -use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampled_token_section::SampledTokenSection; use llama_cpp_bindings_tests::ingested_generation::ingested_generation; @@ -35,8 +33,7 @@ fn qwen35_classifier_ends_generation_without_emitting_the_end_of_generation_toke const GENERATED_TEXT: &str = "The answer is four."; let session = user_turn_chat_session(fixture.model, "What is 2 + 2?", "[]", false)?; - let mut classifier = - SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let mut classifier = SampledTokenClassifier::for_chat_session(&session); assert_eq!( visible_text(&ingested_generation( @@ -73,8 +70,7 @@ fn an_answer_stopped_inside_unclosed_reasoning_releases_its_unsettled_tail_as_co const UNSETTLED_TAIL: &str = ", -) -> Result<()> { - const GENERATED_TEXT: &str = r#"{"answer": 4"#; - - let session = user_turn_chat_session(fixture.model, "What is 2 + 2?", "[]", false)?; - let mut classifier = - SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); - let outcomes = ingested_generation(&mut classifier, fixture.model, GENERATED_TEXT); - - assert!(!session.supports_tool_calls()); - assert_eq!(visible_text(&outcomes), GENERATED_TEXT); - assert!( - outcomes - .iter() - .all(|outcome| matches!(outcome.sampled_token, SampledToken::Content(_))), - "{outcomes:?}" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn gemma3_attributes_a_bare_json_tool_call_object_to_the_tool_call( +fn gemma3_whose_template_cannot_render_tool_calls_keeps_a_bare_json_tool_call_as_content( fixture: &LlamaFixture<'_>, ) -> Result<()> { const GENERATED_TEXT: &str = r#"{"name": "get_weather", "arguments": {"location": "Paris"}}"#; let session = user_turn_chat_session(fixture.model, "What is 2 + 2?", "[]", false)?; - let mut classifier = - SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let mut classifier = SampledTokenClassifier::for_chat_session(&session); let outcomes = ingested_generation(&mut classifier, fixture.model, GENERATED_TEXT); - assert!( - outcomes - .iter() - .all(|outcome| matches!(outcome.sampled_token, SampledToken::ToolCall(_))), - "{outcomes:?}" + assert_eq!( + [ + visible_text_in(&outcomes, SampledTokenSection::Reasoning), + visible_text_in(&outcomes, SampledTokenSection::Content), + visible_text_in(&outcomes, SampledTokenSection::ToolCall), + visible_text_in(&outcomes, SampledTokenSection::Pending), + ], + [ + String::new(), + GENERATED_TEXT.to_owned(), + String::new(), + String::new() + ] ); assert_eq!( - classifier.usage().tool_call_tokens, + classifier.usage().content_tokens, u64::try_from(outcomes.len())? + END_OF_GENERATION_TOKENS ); diff --git a/llama-cpp-bindings-tests/tests/grammar_sampler.rs b/llama-cpp-bindings-tests/tests/grammar_sampler.rs index 25f07574e..d5ec411f8 100644 --- a/llama-cpp-bindings-tests/tests/grammar_sampler.rs +++ b/llama-cpp-bindings-tests/tests/grammar_sampler.rs @@ -2,7 +2,6 @@ use std::ffi::CString; use anyhow::Result; -use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::error::grammar_error::GrammarError; @@ -152,8 +151,7 @@ fn grammar_constrained_chat_answer_is_yes_or_no_content_that_ends_generation( let tokens = session.tokenize_prompt()?; let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - let mut classifier = - SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let mut classifier = SampledTokenClassifier::for_chat_session(&session); classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; context.decode(&batch)?; diff --git a/llama-cpp-bindings-tests/tests/image_question_answering.rs b/llama-cpp-bindings-tests/tests/image_question_answering.rs index 756f5ef6c..0d28bc531 100644 --- a/llama-cpp-bindings-tests/tests/image_question_answering.rs +++ b/llama-cpp-bindings-tests/tests/image_question_answering.rs @@ -1,7 +1,6 @@ use anyhow::Context; use anyhow::Result; -use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::eval_multimodal_chunks_params::EvalMultimodalChunksParams; @@ -79,8 +78,7 @@ fn a_question_about_an_image_is_answered_with_its_image_tokens_counted_as_input( "vision input must produce at least one image chunk" ); - let mut classifier = - SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let mut classifier = SampledTokenClassifier::for_chat_session(&session); let n_past = classifier .eval_multimodal_chunks( &chunks, diff --git a/llama-cpp-bindings-tests/tests/main.rs b/llama-cpp-bindings-tests/tests/main.rs index 58c7d577e..b3c82f992 100644 --- a/llama-cpp-bindings-tests/tests/main.rs +++ b/llama-cpp-bindings-tests/tests/main.rs @@ -5,7 +5,7 @@ mod chat_prompt_rendering; mod chat_session_input_validation; mod chat_session_llama_cpp_failures; mod chat_template_lookup; -mod chat_template_tool_call_support; +mod chat_template_override; mod completion_usage_accounting; mod context_creation_limits; mod context_dimensions; @@ -14,7 +14,6 @@ mod decode_abort_control; mod decode_batch_rejection; mod detokenization; mod dry_sampler; -mod duck_typed_tool_calls; mod embedding_availability; mod embedding_generation; mod embedding_retrieval_errors; diff --git a/llama-cpp-bindings-tests/tests/multimodal_audio.rs b/llama-cpp-bindings-tests/tests/multimodal_audio.rs index 4de77a03f..9f76c04d0 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_audio.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_audio.rs @@ -1,7 +1,6 @@ use anyhow::Context; use anyhow::Result; -use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings::chat_session_options::ChatSessionOptions; @@ -55,6 +54,7 @@ fn evaluate_audio_prompt<'model, TOutcome>( ]; let session = ChatSession::new( model, + model.chat_templates()?, &messages, "[]", &ChatSessionOptions { @@ -88,8 +88,7 @@ fn evaluate_audio_prompt<'model, TOutcome>( "tokenization should produce at least one chunk" ); - let mut classifier = - SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let mut classifier = SampledTokenClassifier::for_chat_session(&session); let n_past = classifier .eval_multimodal_chunks( &chunks, diff --git a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs index 31780b222..4e4a9ebf7 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs @@ -1,7 +1,6 @@ use anyhow::Context; use anyhow::Result; -use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings::chat_session_options::ChatSessionOptions; @@ -37,6 +36,7 @@ fn describe_image_and_audio_chat_session(model: &LlamaModel) -> Result Result<()> { let session = ChatSession::new( fixture.model, + fixture.model.chat_templates()?, &[LlamaChatMessage::new( "user".to_owned(), SINGLE_WORD_INSTRUCTION.to_owned(), diff --git a/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs b/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs index e88d7e633..1e28edafb 100644 --- a/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs +++ b/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs @@ -2,7 +2,6 @@ use anyhow::Context as _; use anyhow::Result; use anyhow::bail; -use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::error::ingest_error::IngestError; use llama_cpp_bindings::error::parse_chat_message_error::ParseChatMessageError; @@ -60,7 +59,7 @@ use llama_cpp_test_harness_macros::llama_test; )] fn chat_session_classifier_starts_in_the_pending_section(fixture: &LlamaFixture<'_>) -> Result<()> { let session = user_turn_chat_session(fixture.model, "hello", "[]", false)?; - let classifier = SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let classifier = SampledTokenClassifier::for_chat_session(&session); assert_eq!(classifier.current_section(), SampledTokenSection::Pending); Ok(()) @@ -135,8 +134,7 @@ fn undeterminable_classifier_releases_each_token_with_its_visible_and_raw_piece( fn finishing_a_chat_answer_settles_its_pending_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { let model = fixture.model; let session = user_turn_chat_session(model, "hello", "[]", false)?; - let mut classifier = - SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let mut classifier = SampledTokenClassifier::for_chat_session(&session); let [ordinary_token] = model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?[..] else { @@ -243,8 +241,7 @@ fn ingesting_a_token_outside_the_vocabulary_reports_it_in_every_classifier( expected ); assert_eq!( - SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect) - .ingest(token, &mut outcomes), + SampledTokenClassifier::for_chat_session(&session).ingest(token, &mut outcomes), expected ); assert_eq!(outcomes, []); @@ -270,8 +267,7 @@ fn ingesting_a_generated_token_reports_a_classification_llama_cpp_fails( fixture: &LlamaFixture<'_>, ) -> Result<()> { let session = user_turn_chat_session(fixture.model, "Hello!", "[]", false)?; - let mut classifier = - SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let mut classifier = SampledTokenClassifier::for_chat_session(&session); let generated = fixture .model .str_to_token("Hi", AddBos::Never, ParseSpecialTokens::Never)?; @@ -298,8 +294,7 @@ fn ending_the_generation_reports_a_classification_llama_cpp_fails( fixture: &LlamaFixture<'_>, ) -> Result<()> { let session = user_turn_chat_session(fixture.model, "Hello!", "[]", false)?; - let mut classifier = - SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let mut classifier = SampledTokenClassifier::for_chat_session(&session); let end_of_sequence = fixture .model .token_eos() @@ -325,8 +320,7 @@ fn ending_the_generation_reports_a_classification_llama_cpp_fails( )] fn sampling_reports_a_classification_llama_cpp_fails(fixture: &LlamaFixture<'_>) -> Result<()> { let session = user_turn_chat_session(fixture.model, "Hello!", "[]", false)?; - let mut classifier = - SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let mut classifier = SampledTokenClassifier::for_chat_session(&session); let mut context = fixture.build_context()?; let mut batch = LlamaBatch::new(ContextLimits::of(&context)); diff --git a/llama-cpp-bindings-tests/tests/tool_call_classification.rs b/llama-cpp-bindings-tests/tests/tool_call_classification.rs index bec28e91d..b8226e422 100644 --- a/llama-cpp-bindings-tests/tests/tool_call_classification.rs +++ b/llama-cpp-bindings-tests/tests/tool_call_classification.rs @@ -1,6 +1,5 @@ use anyhow::Result; -use llama_cpp_bindings::bare_json_tool_calls::BareJsonToolCalls; use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; use llama_cpp_bindings::ingest_outcome::IngestOutcome; use llama_cpp_bindings::sampled_token::SampledToken; @@ -47,8 +46,7 @@ fn qwen35_reasoning_closed_by_a_tool_call_is_attributed_to_each_section( true, )?; let answer = format!("{REASONING_BEFORE_THE_CALL}{QWEN_XML_PARIS_WEATHER_CALL}"); - let mut classifier = - SampledTokenClassifier::for_chat_session(&session, BareJsonToolCalls::Detect); + let mut classifier = SampledTokenClassifier::for_chat_session(&session); let outcomes = ingested_generation(&mut classifier, fixture.model, &answer); let parsed = recognized_answer(&session, &answer, ChatOutputCompleteness::Finished); diff --git a/llama-cpp-bindings-tests/tests/tool_call_parsing.rs b/llama-cpp-bindings-tests/tests/tool_call_parsing.rs index 1c77a591b..b11e72b59 100644 --- a/llama-cpp-bindings-tests/tests/tool_call_parsing.rs +++ b/llama-cpp-bindings-tests/tests/tool_call_parsing.rs @@ -228,6 +228,7 @@ fn qwen35_parses_a_tool_call_cut_inside_its_arguments_as_a_named_call( fn qwen35_parses_consecutive_tool_calls_in_order(fixture: &LlamaFixture<'_>) -> Result<()> { let session = ChatSession::new( fixture.model, + fixture.model.chat_templates()?, &[LlamaChatMessage::new( "user".to_owned(), "What is the weather in Paris and in Berlin?".to_owned(), diff --git a/llama-cpp-bindings-types/src/bracketed_json_shape.rs b/llama-cpp-bindings-types/src/bracketed_json_shape.rs deleted file mode 100644 index 51b18f4b8..000000000 --- a/llama-cpp-bindings-types/src/bracketed_json_shape.rs +++ /dev/null @@ -1,4 +0,0 @@ -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct BracketedJsonShape { - pub name_args_separator: String, -} diff --git a/llama-cpp-bindings-types/src/json_object_shape.rs b/llama-cpp-bindings-types/src/json_object_shape.rs deleted file mode 100644 index b20a5e204..000000000 --- a/llama-cpp-bindings-types/src/json_object_shape.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct JsonObjectShape { - pub name_field: String, - pub arguments_field: String, -} diff --git a/llama-cpp-bindings-types/src/key_value_xml_tags_shape.rs b/llama-cpp-bindings-types/src/key_value_xml_tags_shape.rs deleted file mode 100644 index 220e94b66..000000000 --- a/llama-cpp-bindings-types/src/key_value_xml_tags_shape.rs +++ /dev/null @@ -1,7 +0,0 @@ -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct KeyValueXmlTagsShape { - pub key_open: String, - pub key_close: String, - pub value_open: String, - pub value_close: String, -} diff --git a/llama-cpp-bindings-types/src/lib.rs b/llama-cpp-bindings-types/src/lib.rs index caaf9ac37..256b5a0ac 100644 --- a/llama-cpp-bindings-types/src/lib.rs +++ b/llama-cpp-bindings-types/src/lib.rs @@ -3,19 +3,11 @@ deny(clippy::unwrap_used, clippy::expect_used, clippy::panic) )] -pub mod bracketed_json_shape; -pub mod json_object_shape; -pub mod key_value_xml_tags_shape; pub mod model_provided_tool_call_id; pub mod model_provided_tool_call_id_error; -pub mod paired_quote_shape; pub mod parsed_chat_message; pub mod parsed_tool_call; pub mod parsed_tool_call_id; pub mod token_usage; pub mod token_usage_error; -pub mod tool_call_args_shape; pub mod tool_call_arguments; -pub mod tool_call_markers; -pub mod tool_call_value_quote; -pub mod xml_tags_shape; diff --git a/llama-cpp-bindings-types/src/paired_quote_shape.rs b/llama-cpp-bindings-types/src/paired_quote_shape.rs deleted file mode 100644 index 1126d3ae6..000000000 --- a/llama-cpp-bindings-types/src/paired_quote_shape.rs +++ /dev/null @@ -1,7 +0,0 @@ -use crate::tool_call_value_quote::ToolCallValueQuote; - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct PairedQuoteShape { - pub name_args_separator: String, - pub value_quote: ToolCallValueQuote, -} diff --git a/llama-cpp-bindings-types/src/tool_call_args_shape.rs b/llama-cpp-bindings-types/src/tool_call_args_shape.rs deleted file mode 100644 index 10f3b1fb4..000000000 --- a/llama-cpp-bindings-types/src/tool_call_args_shape.rs +++ /dev/null @@ -1,14 +0,0 @@ -use crate::bracketed_json_shape::BracketedJsonShape; -use crate::json_object_shape::JsonObjectShape; -use crate::key_value_xml_tags_shape::KeyValueXmlTagsShape; -use crate::paired_quote_shape::PairedQuoteShape; -use crate::xml_tags_shape::XmlTagsShape; - -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum ToolCallArgsShape { - BracketedJson(BracketedJsonShape), - JsonObject(JsonObjectShape), - KeyValueXmlTags(KeyValueXmlTagsShape), - PairedQuote(PairedQuoteShape), - XmlTags(XmlTagsShape), -} diff --git a/llama-cpp-bindings-types/src/tool_call_markers.rs b/llama-cpp-bindings-types/src/tool_call_markers.rs deleted file mode 100644 index 1f6610cd3..000000000 --- a/llama-cpp-bindings-types/src/tool_call_markers.rs +++ /dev/null @@ -1,8 +0,0 @@ -use crate::tool_call_args_shape::ToolCallArgsShape; - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct ToolCallMarkers { - pub open: String, - pub close: String, - pub args_shape: ToolCallArgsShape, -} diff --git a/llama-cpp-bindings-types/src/tool_call_value_quote.rs b/llama-cpp-bindings-types/src/tool_call_value_quote.rs deleted file mode 100644 index aca34cbfa..000000000 --- a/llama-cpp-bindings-types/src/tool_call_value_quote.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct ToolCallValueQuote { - pub open: String, - pub close: String, -} diff --git a/llama-cpp-bindings-types/src/xml_tags_shape.rs b/llama-cpp-bindings-types/src/xml_tags_shape.rs deleted file mode 100644 index c09634bea..000000000 --- a/llama-cpp-bindings-types/src/xml_tags_shape.rs +++ /dev/null @@ -1,7 +0,0 @@ -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct XmlTagsShape { - pub function_open_prefix: String, - pub function_close: String, - pub parameter_open_prefix: String, - pub parameter_close: String, -} diff --git a/llama-cpp-bindings/Cargo.toml b/llama-cpp-bindings/Cargo.toml index c73ccb3eb..bb140b710 100644 --- a/llama-cpp-bindings/Cargo.toml +++ b/llama-cpp-bindings/Cargo.toml @@ -17,9 +17,7 @@ llama-cpp-gbnf = { workspace = true } llama-cpp-log-decoder = { workspace = true } llguidance = { workspace = true } log = { workspace = true } -nom = { workspace = true } once_cell = { workspace = true } -serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } toktrie = { workspace = true } diff --git a/llama-cpp-bindings/src/bare_json_tool_call.rs b/llama-cpp-bindings/src/bare_json_tool_call.rs deleted file mode 100644 index 8efb29c27..000000000 --- a/llama-cpp-bindings/src/bare_json_tool_call.rs +++ /dev/null @@ -1,10 +0,0 @@ -use serde::Deserialize; -use serde::de::IgnoredAny; - -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -pub struct BareJsonToolCall { - pub name: String, - #[serde(rename = "arguments")] - _arguments: Option, -} diff --git a/llama-cpp-bindings/src/bare_json_tool_call_field.rs b/llama-cpp-bindings/src/bare_json_tool_call_field.rs deleted file mode 100644 index 765eb3fe6..000000000 --- a/llama-cpp-bindings/src/bare_json_tool_call_field.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum BareJsonToolCallField { - Arguments, - Name, -} diff --git a/llama-cpp-bindings/src/bare_json_tool_call_probe.rs b/llama-cpp-bindings/src/bare_json_tool_call_probe.rs deleted file mode 100644 index dd1f49552..000000000 --- a/llama-cpp-bindings/src/bare_json_tool_call_probe.rs +++ /dev/null @@ -1,233 +0,0 @@ -use std::mem; - -use crate::bare_json_tool_call_probe_state::BareJsonToolCallProbeState; -use crate::ingest_outcome::IngestOutcome; -use crate::json_probe_outcome::JsonProbeOutcome; -use crate::sampled_token::SampledToken; -use crate::streaming_json_probe::StreamingJsonProbe; -use crate::token_piece::TokenPiece; - -const fn is_visible_content(outcome: &IngestOutcome) -> bool { - matches!(outcome.sampled_token, SampledToken::Content(_)) - && matches!(outcome.piece, TokenPiece::Visible(_)) -} - -/// Holds visible content that opens a bare JSON object until the object settles. -/// -/// A complete `{"name": ..., "arguments": ...}` object is released as tool-call tokens; anything -/// else is released as the content it was. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct BareJsonToolCallProbe { - state: BareJsonToolCallProbeState, -} - -impl Default for BareJsonToolCallProbe { - fn default() -> Self { - Self { - state: BareJsonToolCallProbeState::Idle, - } - } -} - -impl BareJsonToolCallProbe { - pub fn pass(&mut self, outcome: IngestOutcome, released: &mut Vec) { - let is_content = is_visible_content(&outcome); - - match mem::replace(&mut self.state, BareJsonToolCallProbeState::Idle) { - BareJsonToolCallProbeState::Holding { held, .. } if !is_content => { - released.extend(held); - released.push(outcome); - } - BareJsonToolCallProbeState::Holding { - mut probe, - mut held, - } => { - let probe_outcome = probe.feed(outcome.piece.raw()); - - held.push(outcome); - self.settle(probe_outcome, probe, held, released); - } - BareJsonToolCallProbeState::Idle - if is_content && outcome.piece.raw().trim_start().starts_with('{') => - { - let mut probe = StreamingJsonProbe::default(); - let probe_outcome = probe.feed(outcome.piece.raw()); - - self.settle(probe_outcome, probe, vec![outcome], released); - } - BareJsonToolCallProbeState::Idle => released.push(outcome), - } - } - - fn settle( - &mut self, - probe_outcome: JsonProbeOutcome, - probe: StreamingJsonProbe, - held: Vec, - released: &mut Vec, - ) { - match probe_outcome { - JsonProbeOutcome::StillPossiblyValid => { - self.state = BareJsonToolCallProbeState::Holding { probe, held }; - } - JsonProbeOutcome::CompletedValid => { - released.extend(held.into_iter().map(|outcome| IngestOutcome { - sampled_token: SampledToken::ToolCall(outcome.sampled_token.token()), - piece: outcome.piece, - })); - } - JsonProbeOutcome::Failed => released.extend(held), - } - } - - /// Releases every held token as the content it was. - pub fn release(&mut self, released: &mut Vec) { - if let BareJsonToolCallProbeState::Holding { held, .. } = - mem::replace(&mut self.state, BareJsonToolCallProbeState::Idle) - { - released.extend(held); - } - } -} - -#[cfg(test)] -mod tests { - use super::BareJsonToolCallProbe; - use crate::ingest_outcome::IngestOutcome; - use crate::sampled_token::SampledToken; - use crate::token::LlamaToken; - use crate::token_piece::TokenPiece; - - fn content(piece: &str) -> IngestOutcome { - IngestOutcome { - sampled_token: SampledToken::Content(LlamaToken::new(1)), - piece: TokenPiece::Visible(piece.to_owned()), - } - } - - fn tool_call(piece: &str) -> IngestOutcome { - IngestOutcome { - sampled_token: SampledToken::ToolCall(LlamaToken::new(1)), - piece: TokenPiece::Visible(piece.to_owned()), - } - } - - fn pass_all( - probe: &mut BareJsonToolCallProbe, - outcomes: Vec, - ) -> Vec { - let mut released = Vec::new(); - - for outcome in outcomes { - probe.pass(outcome, &mut released); - } - - released - } - - #[test] - fn content_that_does_not_open_an_object_passes_through() { - let mut probe = BareJsonToolCallProbe::default(); - - assert_eq!(pass_all(&mut probe, vec![content("Hi")]), [content("Hi")]); - } - - #[test] - fn a_complete_named_tool_call_object_is_released_as_tool_call_tokens() { - let mut probe = BareJsonToolCallProbe::default(); - - assert_eq!( - pass_all( - &mut probe, - vec![ - content(" {\"name\":"), - content("\"f\",\"arguments\""), - content(":{}}") - ] - ), - [ - tool_call(" {\"name\":"), - tool_call("\"f\",\"arguments\""), - tool_call(":{}}"), - ] - ); - } - - #[test] - fn an_object_that_is_not_a_tool_call_is_released_as_content() { - let mut probe = BareJsonToolCallProbe::default(); - - assert_eq!( - pass_all(&mut probe, vec![content("{\"foo\":"), content("\"bar\"}")]), - [content("{\"foo\":"), content("\"bar\"}")] - ); - } - - #[test] - fn an_unfinished_object_is_held_until_released() { - let mut probe = BareJsonToolCallProbe::default(); - let mut released = pass_all(&mut probe, vec![content("{\"name\":")]); - - assert_eq!(released, []); - - probe.release(&mut released); - - assert_eq!(released, [content("{\"name\":")]); - } - - #[test] - fn a_token_outside_content_releases_the_held_object_before_itself() { - let mut probe = BareJsonToolCallProbe::default(); - let reasoning = IngestOutcome { - sampled_token: SampledToken::Reasoning(LlamaToken::new(2)), - piece: TokenPiece::Visible("hmm".to_owned()), - }; - - assert_eq!( - pass_all(&mut probe, vec![content("{\"name\":"), reasoning.clone()]), - [content("{\"name\":"), reasoning] - ); - } - - #[test] - fn a_content_marker_releases_the_held_object_before_itself() { - let mut probe = BareJsonToolCallProbe::default(); - let marker = IngestOutcome { - sampled_token: SampledToken::Content(LlamaToken::new(3)), - piece: TokenPiece::Marker("<|im_end|>".to_owned()), - }; - - assert_eq!( - pass_all(&mut probe, vec![content("{\"name\":"), marker.clone()]), - [content("{\"name\":"), marker] - ); - } - - #[test] - fn two_consecutive_tool_call_objects_are_recognized_separately() { - let mut probe = BareJsonToolCallProbe::default(); - - assert_eq!( - pass_all( - &mut probe, - vec![ - content("{\"name\":\"f\",\"arguments\":{}}"), - content("{\"name\":\"g\",\"arguments\":{}}"), - ] - ), - [ - tool_call("{\"name\":\"f\",\"arguments\":{}}"), - tool_call("{\"name\":\"g\",\"arguments\":{}}"), - ] - ); - } - - #[test] - fn releasing_an_idle_probe_releases_nothing() { - let mut released = Vec::new(); - - BareJsonToolCallProbe::default().release(&mut released); - - assert_eq!(released, []); - } -} diff --git a/llama-cpp-bindings/src/bare_json_tool_call_probe_state.rs b/llama-cpp-bindings/src/bare_json_tool_call_probe_state.rs deleted file mode 100644 index 72e54df86..000000000 --- a/llama-cpp-bindings/src/bare_json_tool_call_probe_state.rs +++ /dev/null @@ -1,11 +0,0 @@ -use crate::ingest_outcome::IngestOutcome; -use crate::streaming_json_probe::StreamingJsonProbe; - -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum BareJsonToolCallProbeState { - Idle, - Holding { - probe: StreamingJsonProbe, - held: Vec, - }, -} diff --git a/llama-cpp-bindings/src/bare_json_tool_calls.rs b/llama-cpp-bindings/src/bare_json_tool_calls.rs deleted file mode 100644 index 853c839e6..000000000 --- a/llama-cpp-bindings/src/bare_json_tool_calls.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum BareJsonToolCalls { - Detect, - Ignore, -} diff --git a/llama-cpp-bindings/src/chat_message_parse_outcome.rs b/llama-cpp-bindings/src/chat_message_parse_outcome.rs index bfddd1d58..10c277e2a 100644 --- a/llama-cpp-bindings/src/chat_message_parse_outcome.rs +++ b/llama-cpp-bindings/src/chat_message_parse_outcome.rs @@ -1,5 +1,4 @@ use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; -use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; use crate::raw_chat_message::RawChatMessage; @@ -21,41 +20,16 @@ impl ChatMessageParseOutcome { Self::Unrecognized(raw) => Err(raw), } } - - /// The message this answer becomes when `tool_calls` were recognized after it: a recognized - /// answer keeps its content and reasoning, an unrecognized one is kept whole as content. - #[must_use] - pub fn followed_by_tool_calls(self, tool_calls: Vec) -> ParsedChatMessage { - match self { - Self::Recognized(message) => { - ParsedChatMessage::new(message.content, message.reasoning_content, tool_calls) - } - Self::Unrecognized(raw) => ParsedChatMessage::new(raw.text, String::new(), tool_calls), - } - } } #[cfg(test)] mod tests { - use serde_json::json; - use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; - use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; - use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; - use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; use crate::chat_message_parse_outcome::ChatMessageParseOutcome; use crate::chat_output_completeness::ChatOutputCompleteness; use crate::raw_chat_message::RawChatMessage; - fn weather_call() -> ParsedToolCall { - ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - "get_weather".to_owned(), - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ) - } - fn odd_answer() -> RawChatMessage { RawChatMessage { text: "Paris.".to_owned(), @@ -64,40 +38,6 @@ mod tests { } } - #[test] - fn a_recognized_answer_keeps_its_content_and_reasoning_before_the_tool_calls() { - assert_eq!( - ChatMessageParseOutcome::Recognized(ParsedChatMessage::new( - "Checking.".to_owned(), - "The user wants the weather.".to_owned(), - Vec::new(), - )) - .followed_by_tool_calls(vec![weather_call()]), - ParsedChatMessage::new( - "Checking.".to_owned(), - "The user wants the weather.".to_owned(), - vec![weather_call()], - ) - ); - } - - #[test] - fn an_unrecognized_answer_is_kept_whole_as_content_before_the_tool_calls() { - assert_eq!( - ChatMessageParseOutcome::Unrecognized(RawChatMessage { - text: "Checking.".to_owned(), - completeness: ChatOutputCompleteness::Finished, - ffi_error_message: "the format differs".to_owned(), - }) - .followed_by_tool_calls(vec![weather_call()]), - ParsedChatMessage::new( - "Checking.".to_owned(), - String::new(), - vec![weather_call()] - ) - ); - } - #[test] fn a_recognized_answer_is_its_message() { let message = ParsedChatMessage::new("Paris.".to_owned(), String::new(), Vec::new()); diff --git a/llama-cpp-bindings/src/chat_session.rs b/llama-cpp-bindings/src/chat_session.rs index 299f95741..d8c8c3e21 100644 --- a/llama-cpp-bindings/src/chat_session.rs +++ b/llama-cpp-bindings/src/chat_session.rs @@ -7,7 +7,6 @@ use std::slice; use serde_json::Value; use serde_json::json; -use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; @@ -24,7 +23,7 @@ use crate::chat_session_options::ChatSessionOptions; use crate::chat_span::ChatSpan; use crate::chat_span_kind::ChatSpanKind; use crate::chat_spans::ChatSpans; -use crate::chat_template_tool_calls::known_marker_candidates::known_marker_candidates; +use crate::chat_templates::ChatTemplates; use crate::cpp_text_view::cpp_text_view; use crate::error::chat_session_error::ChatSessionError; use crate::error::parse_chat_message_error::ParseChatMessageError; @@ -37,8 +36,6 @@ use crate::parsed_chat::take_parsed_chat_message; use crate::raw_chat_message::RawChatMessage; use crate::sampling::LlamaSampler; use crate::token::LlamaToken; -use crate::tool_call_format::tool_call_format_outcome::ToolCallFormatOutcome; -use crate::tool_call_format::try_parse; const GRAMMAR_ROOT: &str = "root"; @@ -328,26 +325,24 @@ fn chat_session_classify_status_to_result( pub struct ChatSession<'model> { model: &'model LlamaModel, session: NonNull, - supports_tool_calls: bool, } unsafe impl Send for ChatSession<'_> {} impl<'model> ChatSession<'model> { - /// Renders `messages` with `tools_json`, an `OpenAI` tools array, through the model's chat - /// template. + /// Renders `messages` with `tools_json`, an `OpenAI` tools array, through `templates`: the + /// model's own [`LlamaModel::chat_templates`], or [`ChatTemplates::with_override`]. /// /// # Errors /// - /// Returns the [`ChatSessionError`] describing why the model's chat template cannot serve the - /// request. + /// Returns the [`ChatSessionError`] describing why the chat template cannot serve the request. pub fn new( model: &'model LlamaModel, + templates: &ChatTemplates, messages: &[LlamaChatMessage], tools_json: &str, options: &ChatSessionOptions, ) -> Result { - let templates = model.chat_templates()?; let messages_json = messages_json(messages, &options.assistant_turn)?; let tools_json = CString::new(tools_json).map_err(ChatSessionError::RequestContainsNul)?; let mut out_session: llama_cpp_bindings_sys::llama_rs_chat_session_handle = ptr::null_mut(); @@ -364,11 +359,8 @@ impl<'model> ChatSession<'model> { ) }; - chat_session_create_status_to_result(status, out_session, out_error).map(|session| Self { - model, - session, - supports_tool_calls: templates.supports_tool_calls(), - }) + chat_session_create_status_to_result(status, out_session, out_error) + .map(|session| Self { model, session }) } fn text_of( @@ -601,13 +593,6 @@ impl<'model> ChatSession<'model> { ) } - /// Whether the template this request rendered with can render tool calls. When it cannot, - /// [`Self::parse`] recognizes tool calls in the formats other templates use. - #[must_use] - pub const fn supports_tool_calls(&self) -> bool { - self.supports_tool_calls - } - #[must_use] pub const fn model(&self) -> &'model LlamaModel { self.model @@ -650,7 +635,13 @@ impl<'model> ChatSession<'model> { )?)) } - fn parse_with_template( + /// Parses the model's answer with the template's parser. + /// + /// # Errors + /// + /// Returns [`ParseChatMessageError`] when llama.cpp fails while parsing; an answer that does + /// not match the template's format is [`ChatMessageParseOutcome::Unrecognized`]. + pub fn parse( &self, output: &ChatOutput, completeness: ChatOutputCompleteness, @@ -672,50 +663,6 @@ impl<'model> ChatSession<'model> { chat_session_parse_status_to_result(status, out_handle, out_error, output, completeness) } - fn parse_duck_typed_tool_calls( - &self, - output: &ChatOutput, - completeness: ChatOutputCompleteness, - ) -> Result, ParseChatMessageError> { - for candidate in known_marker_candidates() { - if let ToolCallFormatOutcome::Parsed(tool_calls) = try_parse(output.text(), &candidate) - { - let before_tool_calls = output - .text() - .find(&candidate.open) - .map_or_else(|| output.text(), |open| &output.text()[..open]); - - return Ok(Some( - self.parse_with_template(&ChatOutput::from(before_tool_calls), completeness)? - .followed_by_tool_calls(tool_calls), - )); - } - } - - Ok(None) - } - - /// Parses the model's answer with the template's parser. When the template cannot render - /// tool calls, tool calls in the formats other templates use are recognized first. - /// - /// # Errors - /// - /// Returns [`ParseChatMessageError`] when llama.cpp fails while parsing; an answer that does - /// not match the template's format is [`ChatMessageParseOutcome::Unrecognized`]. - pub fn parse( - &self, - output: &ChatOutput, - completeness: ChatOutputCompleteness, - ) -> Result { - if !self.supports_tool_calls - && let Some(duck_typed) = self.parse_duck_typed_tool_calls(output, completeness)? - { - return Ok(ChatMessageParseOutcome::Recognized(duck_typed)); - } - - self.parse_with_template(output, completeness) - } - /// Returns how the template's parser attributes the bytes of the model's answer to /// reasoning, content, and tool calls. /// diff --git a/llama-cpp-bindings/src/chat_session_attribution.rs b/llama-cpp-bindings/src/chat_session_attribution.rs index 74f3ed63f..da6b294ef 100644 --- a/llama-cpp-bindings/src/chat_session_attribution.rs +++ b/llama-cpp-bindings/src/chat_session_attribution.rs @@ -2,7 +2,6 @@ use std::collections::VecDeque; use crate::chat_output::ChatOutput; use crate::chat_session::ChatSession; -use crate::content_probe::ContentProbe; use crate::pending_chat_piece::PendingChatPiece; use crate::token::LlamaToken; @@ -11,5 +10,4 @@ pub struct ChatSessionAttribution<'session, 'model> { pub preserved_tokens: Vec, pub output: ChatOutput, pub pending_pieces: VecDeque, - pub content_probe: ContentProbe, } diff --git a/llama-cpp-bindings/src/chat_template_tool_calls.rs b/llama-cpp-bindings/src/chat_template_tool_calls.rs deleted file mode 100644 index cd9732918..000000000 --- a/llama-cpp-bindings/src/chat_template_tool_calls.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub mod gemma4_call_block; -pub mod glm47_key_value_tags; -pub mod known_marker_candidates; -pub mod mistral3_arrow_args; -pub mod qwen3_json_inside_tool_call; -pub mod qwen_xml_tags; diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/gemma4_call_block.rs b/llama-cpp-bindings/src/chat_template_tool_calls/gemma4_call_block.rs deleted file mode 100644 index 39603a26f..000000000 --- a/llama-cpp-bindings/src/chat_template_tool_calls/gemma4_call_block.rs +++ /dev/null @@ -1,23 +0,0 @@ -use llama_cpp_bindings_types::paired_quote_shape::PairedQuoteShape; -use llama_cpp_bindings_types::tool_call_args_shape::ToolCallArgsShape; -use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; -use llama_cpp_bindings_types::tool_call_value_quote::ToolCallValueQuote; - -pub struct Gemma4CallBlockFormat; - -impl Gemma4CallBlockFormat { - #[must_use] - pub fn markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "<|tool_call>call:".to_owned(), - close: "}".to_owned(), - args_shape: ToolCallArgsShape::PairedQuote(PairedQuoteShape { - name_args_separator: "{".to_owned(), - value_quote: ToolCallValueQuote { - open: "<|\"|>".to_owned(), - close: "<|\"|>".to_owned(), - }, - }), - } - } -} diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/glm47_key_value_tags.rs b/llama-cpp-bindings/src/chat_template_tool_calls/glm47_key_value_tags.rs deleted file mode 100644 index fb2e77e8f..000000000 --- a/llama-cpp-bindings/src/chat_template_tool_calls/glm47_key_value_tags.rs +++ /dev/null @@ -1,21 +0,0 @@ -use llama_cpp_bindings_types::key_value_xml_tags_shape::KeyValueXmlTagsShape; -use llama_cpp_bindings_types::tool_call_args_shape::ToolCallArgsShape; -use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; - -pub struct Glm47KeyValueTagsFormat; - -impl Glm47KeyValueTagsFormat { - #[must_use] - pub fn markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "".to_owned(), - close: "".to_owned(), - args_shape: ToolCallArgsShape::KeyValueXmlTags(KeyValueXmlTagsShape { - key_open: "".to_owned(), - key_close: "".to_owned(), - value_open: "".to_owned(), - value_close: "".to_owned(), - }), - } - } -} diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/known_marker_candidates.rs b/llama-cpp-bindings/src/chat_template_tool_calls/known_marker_candidates.rs deleted file mode 100644 index 4a60ca40b..000000000 --- a/llama-cpp-bindings/src/chat_template_tool_calls/known_marker_candidates.rs +++ /dev/null @@ -1,132 +0,0 @@ -use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; - -use crate::chat_template_tool_calls::gemma4_call_block::Gemma4CallBlockFormat; -use crate::chat_template_tool_calls::glm47_key_value_tags::Glm47KeyValueTagsFormat; -use crate::chat_template_tool_calls::mistral3_arrow_args::Mistral3ArrowArgsFormat; -use crate::chat_template_tool_calls::qwen_xml_tags::QwenXmlTagsFormat; -use crate::chat_template_tool_calls::qwen3_json_inside_tool_call::Qwen3JsonInsideToolCallFormat; - -/// Every chat-template tool-call format this crate knows, ordered so that the more restrictive -/// shapes are tried before the ones whose separators could greedily match them. -#[must_use] -pub fn known_marker_candidates() -> Vec { - vec![ - Qwen3JsonInsideToolCallFormat::markers(), - QwenXmlTagsFormat::markers(), - Glm47KeyValueTagsFormat::markers(), - Mistral3ArrowArgsFormat::markers(), - Gemma4CallBlockFormat::markers(), - ] -} - -#[cfg(test)] -mod tests { - use serde_json::json; - - use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; - use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; - use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; - use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; - - use super::known_marker_candidates; - use crate::chat_template_tool_calls::gemma4_call_block::Gemma4CallBlockFormat; - use crate::chat_template_tool_calls::glm47_key_value_tags::Glm47KeyValueTagsFormat; - use crate::chat_template_tool_calls::mistral3_arrow_args::Mistral3ArrowArgsFormat; - use crate::chat_template_tool_calls::qwen_xml_tags::QwenXmlTagsFormat; - use crate::tool_call_format::tool_call_format_outcome::ToolCallFormatOutcome; - use crate::tool_call_format::try_parse; - - fn first_candidate_parsing(body: &str) -> Option { - known_marker_candidates().into_iter().find(|candidate| { - matches!(try_parse(body, candidate), ToolCallFormatOutcome::Parsed(_)) - }) - } - - fn weather_in_paris() -> ToolCallFormatOutcome { - ToolCallFormatOutcome::Parsed(vec![ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - "get_weather".to_owned(), - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - )]) - } - - #[test] - fn no_known_candidate_parses_plain_content() { - let plain_content = "Sorry, I cannot help with that request."; - - for candidate in known_marker_candidates() { - assert_eq!( - try_parse(plain_content, &candidate), - ToolCallFormatOutcome::NoMatch, - "expected NoMatch for plain content under candidate {candidate:?}" - ); - } - } - - #[test] - fn qwen_xml_input_resolves_to_the_qwen_xml_tags_format() { - let qwen_input = "\n\ - \n\ - \n\ - Paris\n\ - \n\ - \n\ - "; - let resolved = first_candidate_parsing(qwen_input); - - assert_eq!( - resolved, - Some(QwenXmlTagsFormat::markers()), - "duck-type ordering must resolve Qwen XML via the XmlTags shape (most restrictive \ - shape that requires `Paris<|\"|>}"; - let resolved = first_candidate_parsing(gemma_input); - - assert_eq!(resolved, Some(Gemma4CallBlockFormat::markers())); - assert_eq!( - resolved.map(|candidate| try_parse(gemma_input, &candidate)), - Some(weather_in_paris()) - ); - } -} diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/mistral3_arrow_args.rs b/llama-cpp-bindings/src/chat_template_tool_calls/mistral3_arrow_args.rs deleted file mode 100644 index c333e250d..000000000 --- a/llama-cpp-bindings/src/chat_template_tool_calls/mistral3_arrow_args.rs +++ /dev/null @@ -1,18 +0,0 @@ -use llama_cpp_bindings_types::bracketed_json_shape::BracketedJsonShape; -use llama_cpp_bindings_types::tool_call_args_shape::ToolCallArgsShape; -use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; - -pub struct Mistral3ArrowArgsFormat; - -impl Mistral3ArrowArgsFormat { - #[must_use] - pub fn markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "[TOOL_CALLS]".to_owned(), - close: String::new(), - args_shape: ToolCallArgsShape::BracketedJson(BracketedJsonShape { - name_args_separator: "[ARGS]".to_owned(), - }), - } - } -} diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/qwen3_json_inside_tool_call.rs b/llama-cpp-bindings/src/chat_template_tool_calls/qwen3_json_inside_tool_call.rs deleted file mode 100644 index 1aae1f58b..000000000 --- a/llama-cpp-bindings/src/chat_template_tool_calls/qwen3_json_inside_tool_call.rs +++ /dev/null @@ -1,19 +0,0 @@ -use llama_cpp_bindings_types::json_object_shape::JsonObjectShape; -use llama_cpp_bindings_types::tool_call_args_shape::ToolCallArgsShape; -use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; - -pub struct Qwen3JsonInsideToolCallFormat; - -impl Qwen3JsonInsideToolCallFormat { - #[must_use] - pub fn markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "".to_owned(), - close: "".to_owned(), - args_shape: ToolCallArgsShape::JsonObject(JsonObjectShape { - name_field: "name".to_owned(), - arguments_field: "arguments".to_owned(), - }), - } - } -} diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/qwen_xml_tags.rs b/llama-cpp-bindings/src/chat_template_tool_calls/qwen_xml_tags.rs deleted file mode 100644 index 5db3cd41e..000000000 --- a/llama-cpp-bindings/src/chat_template_tool_calls/qwen_xml_tags.rs +++ /dev/null @@ -1,21 +0,0 @@ -use llama_cpp_bindings_types::tool_call_args_shape::ToolCallArgsShape; -use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; -use llama_cpp_bindings_types::xml_tags_shape::XmlTagsShape; - -pub struct QwenXmlTagsFormat; - -impl QwenXmlTagsFormat { - #[must_use] - pub fn markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "".to_owned(), - close: "".to_owned(), - args_shape: ToolCallArgsShape::XmlTags(XmlTagsShape { - function_open_prefix: "".to_owned(), - parameter_open_prefix: "".to_owned(), - }), - } - } -} diff --git a/llama-cpp-bindings/src/chat_templates.rs b/llama-cpp-bindings/src/chat_templates.rs index 70830b31c..4b28abe70 100644 --- a/llama-cpp-bindings/src/chat_templates.rs +++ b/llama-cpp-bindings/src/chat_templates.rs @@ -1,3 +1,4 @@ +use std::ffi::CStr; use std::ffi::c_char; use std::ptr; use std::ptr::NonNull; @@ -8,6 +9,7 @@ use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use crate::error::chat_templates_error::ChatTemplatesError; use crate::model::LlamaModel; +use crate::model::llama_chat_template::LlamaChatTemplate; fn chat_templates_init_status_to_result( status: llama_cpp_bindings_sys::llama_rs_chat_templates_init_status, @@ -49,6 +51,13 @@ fn chat_templates_init_status_to_result( } .into()) } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_TEMPLATE_OVERRIDE_ARG => { + Err(FfiContractError { + operation: "llama_rs_chat_templates_init", + detail: "was given a null template_override argument", + } + .into()) + } llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_TEMPLATES_ARG => { Err(FfiContractError { operation: "llama_rs_chat_templates_init", @@ -71,8 +80,8 @@ fn chat_templates_init_status_to_result( } } -/// The chat templates of a model as llama.cpp prepares them once per model: with the model's -/// BOS and EOS, the template variant for tool calls, and llama.cpp's template fixes applied. +/// Chat templates as llama.cpp prepares them for a model: with the model's BOS and EOS, the +/// template variant for tool calls, and llama.cpp's template fixes applied. #[derive(Debug)] pub struct ChatTemplates { templates: NonNull, @@ -82,17 +91,14 @@ unsafe impl Send for ChatTemplates {} unsafe impl Sync for ChatTemplates {} impl ChatTemplates { - /// # Errors - /// - /// Returns [`ChatTemplatesError::ModelHasNoChatTemplate`] for a model without a chat template, - /// or another [`ChatTemplatesError`] when llama.cpp cannot prepare the templates. - pub fn new(model: &LlamaModel) -> Result { + fn init(model: &LlamaModel, template_override: &CStr) -> Result { let mut out_templates: llama_cpp_bindings_sys::llama_rs_chat_templates_handle = ptr::null_mut(); let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_chat_templates_init( model.model.as_ptr(), + template_override.as_ptr(), &raw mut out_templates, &raw mut out_error, ) @@ -102,20 +108,37 @@ impl ChatTemplates { .map(|templates| Self { templates }) } - #[must_use] - pub const fn as_ptr(&self) -> llama_cpp_bindings_sys::llama_rs_chat_templates_handle { - self.templates.as_ptr() + /// The model's own chat templates. + /// + /// # Errors + /// + /// Returns [`ChatTemplatesError::ModelHasNoChatTemplate`] for a model without a chat template, + /// or another [`ChatTemplatesError`] when llama.cpp cannot prepare the templates. + pub fn new(model: &LlamaModel) -> Result { + Self::init(model, c"") } - /// Whether the template a tool call request renders with can render tool calls, as llama.cpp - /// detects it from the template program. - #[must_use] - pub fn supports_tool_calls(&self) -> bool { - unsafe { - llama_cpp_bindings_sys::llama_rs_chat_templates_supports_tool_calls( - self.templates.as_ptr(), - ) + /// `template` in place of the model's own chat templates, the way llama.cpp serves a model + /// whose template cannot render a request, such as tool calls. + /// + /// # Errors + /// + /// Returns [`ChatTemplatesError::EmptyTemplateOverride`] for an empty `template`, or another + /// [`ChatTemplatesError`] when llama.cpp cannot prepare the template. + pub fn with_override( + model: &LlamaModel, + template: &LlamaChatTemplate, + ) -> Result { + if template.as_c_str().is_empty() { + return Err(ChatTemplatesError::EmptyTemplateOverride); } + + Self::init(model, template.as_c_str()) + } + + #[must_use] + pub const fn as_ptr(&self) -> llama_cpp_bindings_sys::llama_rs_chat_templates_handle { + self.templates.as_ptr() } } @@ -178,6 +201,10 @@ mod tests { llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_MODEL_ARG, ptr::null_mut() ), + init_error( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_TEMPLATE_OVERRIDE_ARG, + ptr::null_mut() + ), init_error( llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_TEMPLATES_ARG, ptr::null_mut() @@ -197,6 +224,7 @@ mod tests { message: "the template threw".to_owned() }), Some(contract("was given a null model argument")), + Some(contract("was given a null template_override argument")), Some(contract("was given a null out_templates argument")), Some(contract("was given a null out_error argument")), Some(ChatTemplatesError::FfiStatus(FfiStatusError { diff --git a/llama-cpp-bindings/src/content_probe.rs b/llama-cpp-bindings/src/content_probe.rs deleted file mode 100644 index 8a7d6a017..000000000 --- a/llama-cpp-bindings/src/content_probe.rs +++ /dev/null @@ -1,23 +0,0 @@ -use crate::bare_json_tool_call_probe::BareJsonToolCallProbe; -use crate::ingest_outcome::IngestOutcome; - -pub enum ContentProbe { - Disabled, - BareJson(BareJsonToolCallProbe), -} - -impl ContentProbe { - pub fn pass(&mut self, outcome: IngestOutcome, released: &mut Vec) { - match self { - Self::Disabled => released.push(outcome), - Self::BareJson(probe) => probe.pass(outcome, released), - } - } - - pub fn release(&mut self, released: &mut Vec) { - match self { - Self::Disabled => {} - Self::BareJson(probe) => probe.release(released), - } - } -} diff --git a/llama-cpp-bindings/src/error.rs b/llama-cpp-bindings/src/error.rs index 0dc0c7d6e..7e6666cdc 100644 --- a/llama-cpp-bindings/src/error.rs +++ b/llama-cpp-bindings/src/error.rs @@ -1,5 +1,4 @@ pub mod batch_transfer_error; -pub mod bracketed_args_failure; pub mod chat_session_error; pub mod chat_template_error; pub mod chat_templates_error; @@ -11,9 +10,7 @@ pub mod fit_error; pub mod grammar_error; pub mod grammar_runtime_error; pub mod ingest_error; -pub mod json_object_failure; pub mod json_schema_to_grammar_error; -pub mod key_value_xml_tags_failure; pub mod llama_context_load_error; pub mod llama_cpp_error; pub mod llama_lora_adapter_init_error; @@ -28,7 +25,6 @@ pub mod memory_seq_rm_error; pub mod meta_val_error; pub mod model_params_error; pub mod new_llama_chat_message_error; -pub mod paired_quote_failure; pub mod parse_chat_message_error; pub mod process_failure; pub mod sample_error; @@ -40,8 +36,6 @@ pub mod string_to_token_error; pub mod token_attr_error; pub mod token_outside_vocabulary; pub mod token_to_string_error; -pub mod tool_call_format_failure; -pub mod xml_function_tags_failure; use llama_cpp_error::LlamaCppError; diff --git a/llama-cpp-bindings/src/error/bracketed_args_failure.rs b/llama-cpp-bindings/src/error/bracketed_args_failure.rs deleted file mode 100644 index 4be4e8037..000000000 --- a/llama-cpp-bindings/src/error/bracketed_args_failure.rs +++ /dev/null @@ -1,7 +0,0 @@ -#[derive(Debug, Eq, PartialEq, thiserror::Error)] -pub enum BracketedArgsFailure { - #[error("tool call '{tool_name}' arguments are not valid JSON: {message}")] - InvalidJsonArguments { tool_name: String, message: String }, - #[error("tool call '{tool_name}' arguments truncated before JSON value completed")] - UnterminatedArguments { tool_name: String }, -} diff --git a/llama-cpp-bindings/src/error/chat_templates_error.rs b/llama-cpp-bindings/src/error/chat_templates_error.rs index 539949e33..a8edbcd7a 100644 --- a/llama-cpp-bindings/src/error/chat_templates_error.rs +++ b/llama-cpp-bindings/src/error/chat_templates_error.rs @@ -9,6 +9,10 @@ pub enum ChatTemplatesError { FfiContract(#[from] FfiContractError), #[error("the model has no chat template")] ModelHasNoChatTemplate, + #[error( + "an override template must not be empty, because llama.cpp reads an empty override as no override" + )] + EmptyTemplateOverride, #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/error/json_object_failure.rs b/llama-cpp-bindings/src/error/json_object_failure.rs deleted file mode 100644 index c3bdc56f3..000000000 --- a/llama-cpp-bindings/src/error/json_object_failure.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[derive(Debug, PartialEq, Eq, thiserror::Error)] -pub enum JsonObjectFailure { - #[error("tool call body has malformed JSON: {message}")] - InvalidJson { message: String }, -} diff --git a/llama-cpp-bindings/src/error/key_value_xml_tags_failure.rs b/llama-cpp-bindings/src/error/key_value_xml_tags_failure.rs deleted file mode 100644 index 75960f526..000000000 --- a/llama-cpp-bindings/src/error/key_value_xml_tags_failure.rs +++ /dev/null @@ -1,30 +0,0 @@ -#[derive(Debug, PartialEq, Eq, thiserror::Error)] -pub enum KeyValueXmlTagsFailure { - #[error("tool call function tag has empty name")] - EmptyFunctionName, - #[error("tool call function block is missing close tag '{expected_close}'")] - UnclosedFunctionBlock { expected_close: String }, - #[error("tool call function '{function_name}' has key tag with empty content")] - EmptyKey { function_name: String }, - #[error("tool call function '{function_name}' is missing key close tag '{expected_close}'")] - UnclosedKeyTag { - function_name: String, - expected_close: String, - }, - #[error( - "tool call function '{function_name}' key '{key}' is missing value open tag '{expected_open}'" - )] - MissingValueTag { - function_name: String, - key: String, - expected_open: String, - }, - #[error( - "tool call function '{function_name}' key '{key}' is missing value close tag '{expected_close}'" - )] - UnclosedValueTag { - function_name: String, - key: String, - expected_close: String, - }, -} diff --git a/llama-cpp-bindings/src/error/paired_quote_failure.rs b/llama-cpp-bindings/src/error/paired_quote_failure.rs deleted file mode 100644 index a1d8fc510..000000000 --- a/llama-cpp-bindings/src/error/paired_quote_failure.rs +++ /dev/null @@ -1,22 +0,0 @@ -#[derive(Debug, Eq, PartialEq, thiserror::Error)] -pub enum PairedQuoteFailure { - #[error("empty key in tool call '{tool_name}' arguments")] - EmptyKey { tool_name: String }, - #[error("tool call '{tool_name}' translated arguments are not valid JSON: {message}")] - InvalidJsonArguments { tool_name: String, message: String }, - #[error("tool call '{tool_name}' has unclosed quoted value for key '{key}'")] - UnclosedQuotedValue { tool_name: String, key: String }, - #[error("tool call '{tool_name}' arguments ended without close marker (state: {state})")] - UnclosedArgumentBlock { - tool_name: String, - state: &'static str, - }, - #[error( - "tool call '{tool_name}' has unexpected character '{character}' after value for key '{key}'" - )] - UnexpectedCharAfterValue { - tool_name: String, - key: String, - character: char, - }, -} diff --git a/llama-cpp-bindings/src/error/tool_call_format_failure.rs b/llama-cpp-bindings/src/error/tool_call_format_failure.rs deleted file mode 100644 index 71bbcfbe3..000000000 --- a/llama-cpp-bindings/src/error/tool_call_format_failure.rs +++ /dev/null @@ -1,19 +0,0 @@ -use crate::error::bracketed_args_failure::BracketedArgsFailure; -use crate::error::json_object_failure::JsonObjectFailure; -use crate::error::key_value_xml_tags_failure::KeyValueXmlTagsFailure; -use crate::error::paired_quote_failure::PairedQuoteFailure; -use crate::error::xml_function_tags_failure::XmlFunctionTagsFailure; - -#[derive(Debug, Eq, PartialEq, thiserror::Error)] -pub enum ToolCallFormatFailure { - #[error("bracketed-args parser: {0}")] - BracketedArgs(#[from] BracketedArgsFailure), - #[error("json-object parser: {0}")] - JsonObject(#[from] JsonObjectFailure), - #[error("key-value-xml-tags parser: {0}")] - KeyValueXmlTags(#[from] KeyValueXmlTagsFailure), - #[error("paired-quote parser: {0}")] - PairedQuote(#[from] PairedQuoteFailure), - #[error("xml-function-tags parser: {0}")] - XmlFunctionTags(#[from] XmlFunctionTagsFailure), -} diff --git a/llama-cpp-bindings/src/error/xml_function_tags_failure.rs b/llama-cpp-bindings/src/error/xml_function_tags_failure.rs deleted file mode 100644 index aa8314a7b..000000000 --- a/llama-cpp-bindings/src/error/xml_function_tags_failure.rs +++ /dev/null @@ -1,20 +0,0 @@ -#[derive(Debug, PartialEq, Eq, thiserror::Error)] -pub enum XmlFunctionTagsFailure { - #[error("tool call function tag has empty name")] - EmptyFunctionName, - #[error("tool call function '{function_name}' is missing close tag '{expected_close}'")] - UnclosedFunctionBlock { - function_name: String, - expected_close: String, - }, - #[error("tool call function '{function_name}' has parameter with empty name")] - EmptyParameterName { function_name: String }, - #[error( - "tool call function '{function_name}' parameter '{parameter_name}' is missing close tag '{expected_close}'" - )] - UnclosedParameterBlock { - function_name: String, - parameter_name: String, - expected_close: String, - }, -} diff --git a/llama-cpp-bindings/src/json_probe_outcome.rs b/llama-cpp-bindings/src/json_probe_outcome.rs deleted file mode 100644 index a5b76fe7c..000000000 --- a/llama-cpp-bindings/src/json_probe_outcome.rs +++ /dev/null @@ -1,6 +0,0 @@ -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -pub enum JsonProbeOutcome { - StillPossiblyValid, - CompletedValid, - Failed, -} diff --git a/llama-cpp-bindings/src/lib.rs b/llama-cpp-bindings/src/lib.rs index 6cae4f81a..1f92378e5 100644 --- a/llama-cpp-bindings/src/lib.rs +++ b/llama-cpp-bindings/src/lib.rs @@ -4,11 +4,6 @@ )] mod backend_device_index; -mod bare_json_tool_call; -mod bare_json_tool_call_field; -pub mod bare_json_tool_call_probe; -mod bare_json_tool_call_probe_state; -pub mod bare_json_tool_calls; pub mod batch_add_error; pub mod chat_assistant_prefill; pub mod chat_assistant_turn; @@ -26,11 +21,9 @@ pub mod chat_span; pub mod chat_span_kind; mod chat_span_segment; pub mod chat_spans; -pub mod chat_template_tool_calls; pub mod chat_templates; pub mod chat_tool_choice; pub mod classified_sample; -mod content_probe; pub mod context; pub mod cpp_text_view; pub mod dry_sequence_breakers; @@ -45,7 +38,6 @@ pub mod grammar_matcher; mod grammar_matcher_step_outcome; pub mod ingest_outcome; pub mod invalid_numa_strategy; -pub mod json_probe_outcome; pub mod json_schema_to_grammar; pub mod lazy_grammar_triggers; pub mod llama_backend; @@ -79,9 +71,6 @@ pub mod sampled_token_section; pub mod sampling; pub mod sanitized_grammar; pub mod send_logs_to_log; -pub mod streaming_json_probe; -mod streaming_json_probe_state; pub mod token; mod token_attribution; pub mod token_piece; -pub mod tool_call_format; diff --git a/llama-cpp-bindings/src/sampled_token_classifier.rs b/llama-cpp-bindings/src/sampled_token_classifier.rs index 8c17a0606..749538115 100644 --- a/llama-cpp-bindings/src/sampled_token_classifier.rs +++ b/llama-cpp-bindings/src/sampled_token_classifier.rs @@ -5,8 +5,6 @@ use llama_cpp_bindings_sys::llama_seq_id; use llama_cpp_bindings_types::token_usage::TokenUsage; use llama_cpp_bindings_types::token_usage_error::TokenUsageError; -use crate::bare_json_tool_call_probe::BareJsonToolCallProbe; -use crate::bare_json_tool_calls::BareJsonToolCalls; use crate::batch_add_error::BatchAddError; use crate::chat_output::ChatOutput; use crate::chat_output_completeness::ChatOutputCompleteness; @@ -14,7 +12,6 @@ use crate::chat_piece_role::ChatPieceRole; use crate::chat_session::ChatSession; use crate::chat_session_attribution::ChatSessionAttribution; use crate::classified_sample::ClassifiedSample; -use crate::content_probe::ContentProbe; use crate::context::LlamaContext; use crate::error::eval_multimodal_chunks_error::EvalMultimodalChunksError; use crate::error::ingest_error::IngestError; @@ -75,22 +72,9 @@ impl<'session, 'model> SampledTokenClassifier<'session, 'model> { } } - /// A classifier for the answer to `session`. When the session's template cannot render tool - /// calls and `bare_json_tool_calls` asks for it, content that is a bare JSON tool call object - /// is attributed to a tool call. + /// A classifier for the answer to `session`. #[must_use] - pub fn for_chat_session( - session: &'session ChatSession<'model>, - bare_json_tool_calls: BareJsonToolCalls, - ) -> Self { - let content_probe = if !session.supports_tool_calls() - && bare_json_tool_calls == BareJsonToolCalls::Detect - { - ContentProbe::BareJson(BareJsonToolCallProbe::default()) - } else { - ContentProbe::Disabled - }; - + pub fn for_chat_session(session: &'session ChatSession<'model>) -> Self { Self { model: session.model(), attribution: TokenAttribution::ChatSession(ChatSessionAttribution { @@ -98,7 +82,6 @@ impl<'session, 'model> SampledTokenClassifier<'session, 'model> { preserved_tokens: session.preserved_tokens(), output: ChatOutput::default(), pending_pieces: VecDeque::new(), - content_probe, }), decoder: encoding_rs::UTF_8.new_decoder(), section: SampledTokenSection::Pending, @@ -184,14 +167,7 @@ impl<'session, 'model> SampledTokenClassifier<'session, 'model> { break; }; - attribution.content_probe.pass( - outcome_of(pending_piece.token, role, pending_piece.piece), - &mut released, - ); - } - - if completeness == ChatOutputCompleteness::Finished { - attribution.content_probe.release(&mut released); + released.push(outcome_of(pending_piece.token, role, pending_piece.piece)); } for outcome in released { diff --git a/llama-cpp-bindings/src/streaming_json_probe.rs b/llama-cpp-bindings/src/streaming_json_probe.rs deleted file mode 100644 index 8bf511a08..000000000 --- a/llama-cpp-bindings/src/streaming_json_probe.rs +++ /dev/null @@ -1,379 +0,0 @@ -use std::mem; - -use crate::bare_json_tool_call::BareJsonToolCall; -use crate::json_probe_outcome::JsonProbeOutcome; -use crate::streaming_json_probe_state::StreamingJsonProbeState; - -fn is_named_tool_call(held_text: &str) -> bool { - serde_json::from_str::(held_text) - .is_ok_and(|tool_call| !tool_call.name.is_empty()) -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct StreamingJsonProbe { - held_text: String, - state: StreamingJsonProbeState, -} - -impl Default for StreamingJsonProbe { - fn default() -> Self { - Self { - held_text: String::new(), - state: StreamingJsonProbeState::AwaitingObjectOpen, - } - } -} - -impl StreamingJsonProbe { - pub fn feed(&mut self, piece: &str) -> JsonProbeOutcome { - self.held_text.push_str(piece); - - for character in piece.chars() { - let state = mem::replace(&mut self.state, StreamingJsonProbeState::Failed); - - self.state = state.advance(character); - - if self.state == StreamingJsonProbeState::Failed { - return JsonProbeOutcome::Failed; - } - } - - match self.state { - StreamingJsonProbeState::Closed if is_named_tool_call(&self.held_text) => { - JsonProbeOutcome::CompletedValid - } - StreamingJsonProbeState::Closed | StreamingJsonProbeState::Failed => { - JsonProbeOutcome::Failed - } - _ => JsonProbeOutcome::StillPossiblyValid, - } - } -} - -#[cfg(test)] -mod tests { - use super::StreamingJsonProbe; - use crate::json_probe_outcome::JsonProbeOutcome; - - fn probe(buffer: &str) -> JsonProbeOutcome { - StreamingJsonProbe::default().feed(buffer) - } - - #[test] - fn empty_buffer_is_still_possibly_valid() { - assert_eq!(probe(""), JsonProbeOutcome::StillPossiblyValid,); - } - - #[test] - fn whitespace_only_buffer_is_still_possibly_valid() { - assert_eq!(probe(" \n "), JsonProbeOutcome::StillPossiblyValid,); - } - - #[test] - fn single_open_brace_is_still_possibly_valid() { - assert_eq!(probe("{"), JsonProbeOutcome::StillPossiblyValid,); - } - - #[test] - fn open_brace_with_trailing_space_is_still_possibly_valid() { - assert_eq!(probe("{ "), JsonProbeOutcome::StillPossiblyValid,); - } - - #[test] - fn open_brace_with_quote_starting_key_is_still_possibly_valid() { - assert_eq!(probe(r#"{ ""#), JsonProbeOutcome::StillPossiblyValid,); - } - - #[test] - fn a_name_key_awaiting_its_colon_is_still_possibly_valid() { - assert_eq!(probe(r#"{ "name""#), JsonProbeOutcome::StillPossiblyValid,); - } - - #[test] - fn partial_name_value_quote_is_still_possibly_valid() { - assert_eq!( - probe(r#"{ "name": ""#), - JsonProbeOutcome::StillPossiblyValid, - ); - } - - #[test] - fn partial_name_value_letters_is_still_possibly_valid() { - assert_eq!( - probe(r#"{ "name": "ge"#), - JsonProbeOutcome::StillPossiblyValid, - ); - } - - #[test] - fn complete_name_string_no_comma_is_still_possibly_valid() { - assert_eq!( - probe(r#"{ "name": "get_weather""#), - JsonProbeOutcome::StillPossiblyValid, - ); - } - - #[test] - fn name_then_comma_is_still_possibly_valid() { - assert_eq!( - probe(r#"{ "name": "get_weather","#), - JsonProbeOutcome::StillPossiblyValid, - ); - } - - #[test] - fn name_then_partial_arguments_key_is_still_possibly_valid() { - assert_eq!( - probe(r#"{ "name": "get_weather", "argum"#), - JsonProbeOutcome::StillPossiblyValid, - ); - } - - #[test] - fn name_then_arguments_key_is_still_possibly_valid() { - assert_eq!( - probe(r#"{ "name": "get_weather", "arguments""#), - JsonProbeOutcome::StillPossiblyValid, - ); - } - - #[test] - fn name_then_arguments_open_brace_is_still_possibly_valid() { - assert_eq!( - probe(r#"{ "name": "get_weather", "arguments": {"#), - JsonProbeOutcome::StillPossiblyValid, - ); - } - - #[test] - fn arguments_with_partial_inner_key_value_is_still_possibly_valid() { - assert_eq!( - probe(r#"{ "name": "get_weather", "arguments": {"location":"#), - JsonProbeOutcome::StillPossiblyValid, - ); - } - - #[test] - fn arguments_with_partial_inner_string_value_is_still_possibly_valid() { - assert_eq!( - probe(r#"{ "name": "get_weather", "arguments": {"location": "Pa"#), - JsonProbeOutcome::StillPossiblyValid, - ); - } - - #[test] - fn complete_simple_tool_call_is_completed_valid() { - assert_eq!( - probe(r#"{"name":"f","arguments":{}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn complete_tool_call_with_string_argument_is_completed_valid() { - assert_eq!( - probe(r#"{"name":"get_weather","arguments":{"location":"Paris"}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn complete_tool_call_with_nested_arguments_is_completed_valid() { - assert_eq!( - probe(r#"{"name":"f","arguments":{"a":{"b":[1,2,3]}}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn complete_tool_call_with_close_brace_inside_string_is_completed_valid() { - assert_eq!( - probe(r#"{"name":"f","arguments":{"q":"a } b"}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn complete_tool_call_with_escaped_quotes_in_string_is_completed_valid() { - assert_eq!( - probe(r#"{"name":"f","arguments":{"q":"he said \"hi\""}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn complete_tool_call_with_unicode_strings_is_completed_valid() { - assert_eq!( - probe(r#"{"name":"日本語","arguments":{"city":"パリ"}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn complete_tool_call_with_trailing_whitespace_is_completed_valid() { - assert_eq!( - probe("{\"name\":\"f\",\"arguments\":{}}\n"), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn complete_tool_call_without_arguments_field_is_completed_valid() { - assert_eq!( - probe(r#"{"name":"ping"}"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn a_top_level_value_other_than_an_object_is_failed() { - assert_eq!(probe("["), JsonProbeOutcome::Failed); - } - - #[test] - fn an_unknown_key_is_failed() { - assert_eq!(probe(r#"{"foo":"bar"}"#), JsonProbeOutcome::Failed,); - } - - #[test] - fn a_key_with_an_invalid_escape_is_failed() { - assert_eq!(probe(r#"{"\q":"bar"}"#), JsonProbeOutcome::Failed); - } - - #[test] - fn a_name_value_that_is_not_a_string_is_failed() { - assert_eq!( - probe(r#"{"name":123,"arguments":{}}"#), - JsonProbeOutcome::Failed, - ); - } - - #[test] - fn an_arguments_value_that_is_not_an_object_is_failed() { - assert_eq!( - probe(r#"{"name":"f","arguments":[]}"#), - JsonProbeOutcome::Failed, - ); - } - - #[test] - fn complete_object_with_empty_name_is_failed() { - assert_eq!( - probe(r#"{"name":"","arguments":{}}"#), - JsonProbeOutcome::Failed, - ); - } - - #[test] - fn empty_object_is_failed_due_to_missing_required_name() { - assert_eq!(probe("{}"), JsonProbeOutcome::Failed); - } - - #[test] - fn complete_object_with_arguments_only_no_name_is_failed() { - assert_eq!(probe(r#"{"arguments":{}}"#), JsonProbeOutcome::Failed,); - } - - #[test] - fn leading_whitespace_then_open_brace_is_still_possibly_valid() { - assert_eq!(probe("\n \n{"), JsonProbeOutcome::StillPossiblyValid,); - } - - #[test] - fn leading_whitespace_then_complete_tool_call_is_completed_valid() { - assert_eq!( - probe("\n {\"name\":\"f\",\"arguments\":{}}"), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn a_second_object_after_the_closed_tool_call_is_failed() { - assert_eq!( - probe(r#"{"name":"a","arguments":{}}{"name":"b","arguments":{}}"#), - JsonProbeOutcome::Failed, - ); - } - - #[test] - fn a_partially_written_key_is_still_possibly_valid() { - assert_eq!(probe(r#"{ "n"#), JsonProbeOutcome::StillPossiblyValid,); - } - - #[test] - fn a_comma_before_the_first_key_is_failed() { - assert_eq!(probe("{,}"), JsonProbeOutcome::Failed,); - } - - #[test] - fn key_written_with_an_escape_sequence_is_recognized() { - assert_eq!( - probe(r#"{"na\u006de":"f","arguments":{}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn tool_call_fed_one_character_at_a_time_completes_on_its_last_character() { - let tool_call = r#"{"name":"f","arguments":{"q":"a } b"}}"#; - let mut streaming_probe = StreamingJsonProbe::default(); - let mut outcomes = Vec::new(); - - for character in tool_call.chars() { - outcomes.push(streaming_probe.feed(&character.to_string())); - } - - let (last_outcome, earlier_outcomes) = outcomes - .split_last() - .expect("the tool call must produce outcomes"); - - assert_eq!(*last_outcome, JsonProbeOutcome::CompletedValid); - assert!( - earlier_outcomes - .iter() - .all(|outcome| *outcome == JsonProbeOutcome::StillPossiblyValid) - ); - } - - #[test] - fn syntax_error_inside_arguments_fails_when_the_object_closes() { - let mut streaming_probe = StreamingJsonProbe::default(); - - assert_eq!( - [ - streaming_probe.feed(r#"{"name":"f","arguments":{"q" 1}"#), - streaming_probe.feed("}"), - ], - [ - JsonProbeOutcome::StillPossiblyValid, - JsonProbeOutcome::Failed - ] - ); - } - - #[test] - fn whitespace_around_colons_and_after_values_is_completed_valid() { - assert_eq!( - probe(r#"{"name" : "f" , "arguments" : {} }"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn key_not_followed_by_a_colon_is_failed() { - assert_eq!(probe(r#"{"name""f"}"#), JsonProbeOutcome::Failed); - } - - #[test] - fn value_followed_by_neither_a_comma_nor_a_close_is_failed() { - assert_eq!( - probe(r#"{"name":"f""arguments":{}}"#), - JsonProbeOutcome::Failed - ); - } - - #[test] - fn trailing_comma_before_the_close_is_failed() { - assert_eq!(probe(r#"{"name":"f",}"#), JsonProbeOutcome::Failed); - } -} diff --git a/llama-cpp-bindings/src/streaming_json_probe_state.rs b/llama-cpp-bindings/src/streaming_json_probe_state.rs deleted file mode 100644 index 72564d5ee..000000000 --- a/llama-cpp-bindings/src/streaming_json_probe_state.rs +++ /dev/null @@ -1,157 +0,0 @@ -use crate::bare_json_tool_call_field::BareJsonToolCallField; - -const fn is_json_whitespace(character: char) -> bool { - matches!(character, ' ' | '\t' | '\n' | '\r') -} - -fn field_named(quoted_key: &str) -> Option { - match serde_json::from_str::(quoted_key).ok()?.as_str() { - "arguments" => Some(BareJsonToolCallField::Arguments), - "name" => Some(BareJsonToolCallField::Name), - _ => None, - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum StreamingJsonProbeState { - AwaitingObjectOpen, - AwaitingFirstKeyOrClose, - AwaitingKey, - InKey { - quoted_key: String, - escaped: bool, - }, - AwaitingColon(BareJsonToolCallField), - AwaitingValue(BareJsonToolCallField), - InName { - escaped: bool, - }, - InArguments { - depth: usize, - in_string: bool, - escaped: bool, - }, - AwaitingCommaOrClose, - Closed, - Failed, -} - -impl StreamingJsonProbeState { - pub fn advance(self, character: char) -> Self { - match self { - Self::AwaitingObjectOpen => match character { - '{' => Self::AwaitingFirstKeyOrClose, - _ if is_json_whitespace(character) => Self::AwaitingObjectOpen, - _ => Self::Failed, - }, - Self::AwaitingFirstKeyOrClose => match character { - '"' => Self::InKey { - quoted_key: String::from('"'), - escaped: false, - }, - '}' => Self::Closed, - _ if is_json_whitespace(character) => Self::AwaitingFirstKeyOrClose, - _ => Self::Failed, - }, - Self::AwaitingKey => match character { - '"' => Self::InKey { - quoted_key: String::from('"'), - escaped: false, - }, - _ if is_json_whitespace(character) => Self::AwaitingKey, - _ => Self::Failed, - }, - Self::InKey { - mut quoted_key, - escaped, - } => { - quoted_key.push(character); - - if escaped || character != '"' { - Self::InKey { - quoted_key, - escaped: !escaped && character == '\\', - } - } else { - field_named("ed_key).map_or(Self::Failed, Self::AwaitingColon) - } - } - Self::AwaitingColon(field) => match character { - ':' => Self::AwaitingValue(field), - _ if is_json_whitespace(character) => Self::AwaitingColon(field), - _ => Self::Failed, - }, - Self::AwaitingValue(field) => match (field, character) { - (BareJsonToolCallField::Name, '"') => Self::InName { escaped: false }, - (BareJsonToolCallField::Arguments, '{') => Self::InArguments { - depth: 1, - in_string: false, - escaped: false, - }, - _ if is_json_whitespace(character) => Self::AwaitingValue(field), - _ => Self::Failed, - }, - Self::InName { escaped } => { - if !escaped && character == '"' { - Self::AwaitingCommaOrClose - } else { - Self::InName { - escaped: !escaped && character == '\\', - } - } - } - Self::InArguments { - depth, - in_string, - escaped, - } => Self::advance_arguments(depth, in_string, escaped, character), - Self::AwaitingCommaOrClose => match character { - ',' => Self::AwaitingKey, - '}' => Self::Closed, - _ if is_json_whitespace(character) => Self::AwaitingCommaOrClose, - _ => Self::Failed, - }, - Self::Closed if is_json_whitespace(character) => Self::Closed, - Self::Closed | Self::Failed => Self::Failed, - } - } - - const fn advance_arguments( - depth: usize, - in_string: bool, - escaped: bool, - character: char, - ) -> Self { - if in_string { - return Self::InArguments { - depth, - in_string: escaped || character != '"', - escaped: !escaped && character == '\\', - }; - } - - match character { - '"' => Self::InArguments { - depth, - in_string: true, - escaped: false, - }, - '{' | '[' => Self::InArguments { - depth: depth + 1, - in_string: false, - escaped: false, - }, - '}' | ']' if depth == 1 => Self::AwaitingCommaOrClose, - '}' | ']' => Self::InArguments { - depth: depth - 1, - in_string: false, - escaped: false, - }, - _ => Self::InArguments { - depth, - in_string: false, - escaped: false, - }, - } - } -} diff --git a/llama-cpp-bindings/src/tool_call_format.rs b/llama-cpp-bindings/src/tool_call_format.rs deleted file mode 100644 index d023dac7e..000000000 --- a/llama-cpp-bindings/src/tool_call_format.rs +++ /dev/null @@ -1,236 +0,0 @@ -pub mod bracketed_args; -pub mod consume_optional_prefix; -pub mod json_object; -pub mod key_value_xml_tags; -pub mod paired_quote_args; -pub mod parse_step; -pub mod parsed_prefix; -pub mod scalar_value_to_json; -pub mod separator_split; -pub mod tool_call_format_outcome; -pub mod xml_function_tags; - -use llama_cpp_bindings_types::tool_call_args_shape::ToolCallArgsShape; -use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; - -use crate::error::tool_call_format_failure::ToolCallFormatFailure; -use crate::tool_call_format::tool_call_format_outcome::ToolCallFormatOutcome; - -#[must_use] -pub fn try_parse(body: &str, markers: &ToolCallMarkers) -> ToolCallFormatOutcome { - if markers.open.is_empty() { - return ToolCallFormatOutcome::NoMatch; - } - - let parsed: Result, ToolCallFormatFailure> = match &markers.args_shape { - ToolCallArgsShape::BracketedJson(shape) => { - bracketed_args::parse(body, markers, shape).map_err(Into::into) - } - ToolCallArgsShape::JsonObject(shape) => json_object::parse(body, shape).map_err(Into::into), - ToolCallArgsShape::KeyValueXmlTags(shape) => { - key_value_xml_tags::parse(body, markers, shape).map_err(Into::into) - } - ToolCallArgsShape::PairedQuote(shape) => { - paired_quote_args::parse(body, markers, shape).map_err(Into::into) - } - ToolCallArgsShape::XmlTags(shape) => { - xml_function_tags::parse(body, shape).map_err(Into::into) - } - }; - - match parsed { - Ok(parsed) if parsed.is_empty() => ToolCallFormatOutcome::NoMatch, - Ok(parsed) => ToolCallFormatOutcome::Parsed(parsed), - Err(failure) => ToolCallFormatOutcome::Failed(failure), - } -} - -#[cfg(test)] -mod tests { - use serde_json::json; - - use llama_cpp_bindings_types::bracketed_json_shape::BracketedJsonShape; - use llama_cpp_bindings_types::key_value_xml_tags_shape::KeyValueXmlTagsShape; - use llama_cpp_bindings_types::paired_quote_shape::PairedQuoteShape; - use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; - use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; - use llama_cpp_bindings_types::tool_call_args_shape::ToolCallArgsShape; - use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; - use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; - use llama_cpp_bindings_types::tool_call_value_quote::ToolCallValueQuote; - use llama_cpp_bindings_types::xml_tags_shape::XmlTagsShape; - - use super::ToolCallFormatOutcome; - use super::try_parse; - use crate::error::bracketed_args_failure::BracketedArgsFailure; - use crate::error::tool_call_format_failure::ToolCallFormatFailure; - - fn mistral3_markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "[TOOL_CALLS]".to_owned(), - close: String::new(), - args_shape: ToolCallArgsShape::BracketedJson(BracketedJsonShape { - name_args_separator: "[ARGS]".to_owned(), - }), - } - } - - fn gemma4_markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "<|tool_call>call:".to_owned(), - close: "}".to_owned(), - args_shape: ToolCallArgsShape::PairedQuote(PairedQuoteShape { - name_args_separator: "{".to_owned(), - value_quote: ToolCallValueQuote { - open: "<|\"|>".to_owned(), - close: "<|\"|>".to_owned(), - }, - }), - } - } - - fn qwen35_markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "".to_owned(), - close: "".to_owned(), - args_shape: ToolCallArgsShape::XmlTags(XmlTagsShape { - function_open_prefix: "".to_owned(), - parameter_open_prefix: "".to_owned(), - }), - } - } - - fn glm47_markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "".to_owned(), - close: "".to_owned(), - args_shape: ToolCallArgsShape::KeyValueXmlTags(KeyValueXmlTagsShape { - key_open: "".to_owned(), - key_close: "".to_owned(), - value_open: "".to_owned(), - value_close: "".to_owned(), - }), - } - } - - #[test] - fn dispatches_to_bracketed_args_for_mistral3_shape() { - let outcome = try_parse( - "[TOOL_CALLS]get_weather[ARGS]{\"location\":\"Paris\"}", - &mistral3_markers(), - ); - - assert_eq!( - outcome, - ToolCallFormatOutcome::Parsed(vec![ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - "get_weather".to_owned(), - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - )]), - ); - } - - #[test] - fn dispatches_to_paired_quote_args_for_gemma4_shape() { - let outcome = try_parse( - "<|tool_call>call:get_weather{location:<|\"|>Paris<|\"|>}", - &gemma4_markers(), - ); - - assert_eq!( - outcome, - ToolCallFormatOutcome::Parsed(vec![ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - "get_weather".to_owned(), - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - )]), - ); - } - - #[test] - fn dispatches_to_key_value_xml_tags_for_glm47_shape() { - let outcome = try_parse( - "get_weatherlocationParis", - &glm47_markers(), - ); - - assert_eq!( - outcome, - ToolCallFormatOutcome::Parsed(vec![ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - "get_weather".to_owned(), - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - )]), - ); - } - - #[test] - fn dispatches_to_xml_function_tags_for_qwen35_shape() { - let outcome = try_parse( - "Paris", - &qwen35_markers(), - ); - - assert_eq!( - outcome, - ToolCallFormatOutcome::Parsed(vec![ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - "get_weather".to_owned(), - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - )]), - ); - } - - #[test] - fn no_match_when_open_marker_is_empty() { - let markers = ToolCallMarkers { - open: String::new(), - close: String::new(), - args_shape: ToolCallArgsShape::BracketedJson(BracketedJsonShape { - name_args_separator: "[ARGS]".to_owned(), - }), - }; - - assert_eq!( - try_parse("[TOOL_CALLS]get_weather[ARGS]{}", &markers), - ToolCallFormatOutcome::NoMatch, - ); - } - - #[test] - fn no_match_when_body_lacks_markers() { - assert_eq!( - try_parse("plain text without tool calls", &mistral3_markers()), - ToolCallFormatOutcome::NoMatch, - ); - } - - #[test] - fn failed_when_inner_parser_returns_typed_failure() { - let outcome = try_parse("[TOOL_CALLS]get_weather[ARGS] ", &mistral3_markers()); - - assert_eq!( - outcome, - ToolCallFormatOutcome::Failed(ToolCallFormatFailure::BracketedArgs( - BracketedArgsFailure::UnterminatedArguments { - tool_name: "get_weather".to_owned(), - }, - )), - ); - } - - #[test] - fn try_parse_returns_no_match_for_glm_input_under_qwen_markers() { - let glm_input = "get_weather\ - location\ - Paris\ - "; - - assert_eq!( - try_parse(glm_input, &qwen35_markers()), - ToolCallFormatOutcome::NoMatch, - ); - } -} diff --git a/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs b/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs deleted file mode 100644 index cd9022342..000000000 --- a/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs +++ /dev/null @@ -1,266 +0,0 @@ -use llama_cpp_bindings_types::bracketed_json_shape::BracketedJsonShape; -use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; -use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; -use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; -use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; - -use crate::error::bracketed_args_failure::BracketedArgsFailure; -use crate::tool_call_format::consume_optional_prefix::consume_optional_prefix; -use crate::tool_call_format::parse_step::ParseStep; -use crate::tool_call_format::parsed_prefix::ParsedPrefix; -use crate::tool_call_format::separator_split::SeparatorSplit; - -fn consume_one_json_value<'body>( - input: &'body str, - tool_name: &str, -) -> Result, BracketedArgsFailure> { - let mut stream = serde_json::Deserializer::from_str(input).into_iter::(); - let value = stream - .next() - .ok_or_else(|| BracketedArgsFailure::UnterminatedArguments { - tool_name: tool_name.to_owned(), - })? - .map_err(|err| BracketedArgsFailure::InvalidJsonArguments { - tool_name: tool_name.to_owned(), - message: err.to_string(), - })?; - let consumed = stream.byte_offset(); - - Ok(ParsedPrefix { - value, - remaining: &input[consumed..], - }) -} - -fn parse_one_call<'body>( - input: &'body str, - markers: &ToolCallMarkers, - shape: &BracketedJsonShape, -) -> Result, BracketedArgsFailure> { - if input.is_empty() { - return Ok(ParseStep::Done); - } - - let after_open = consume_optional_prefix(input, markers.open.as_str()); - - let Some(SeparatorSplit { - before: name_raw, - after: after_separator, - }) = SeparatorSplit::at_first(after_open, shape.name_args_separator.as_str()) - else { - return Ok(ParseStep::Done); - }; - - let name = name_raw.trim().to_owned(); - if name.is_empty() { - return Ok(ParseStep::Done); - } - - let ParsedPrefix { - value: arguments_value, - remaining: after_arguments, - } = consume_one_json_value(after_separator, &name)?; - - let after_close = consume_optional_prefix(after_arguments, markers.close.as_str()); - - Ok(ParseStep::Call { - call: ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - name, - ToolCallArguments::ValidJson(arguments_value), - ), - remainder: after_close, - }) -} - -/// # Errors -/// -/// Returns [`BracketedArgsFailure`] when the body looks like a bracketed-JSON -/// tool-call block (matches the name/args separator) but contains a structural -/// issue: invalid JSON arguments or a JSON value truncated mid-stream. -pub fn parse( - body: &str, - markers: &ToolCallMarkers, - shape: &BracketedJsonShape, -) -> Result, BracketedArgsFailure> { - if shape.name_args_separator.is_empty() { - return Ok(Vec::new()); - } - - let mut parsed = Vec::new(); - let mut remaining = body.trim_start(); - - loop { - match parse_one_call(remaining, markers, shape)? { - ParseStep::Done => break, - ParseStep::Call { call, remainder } => { - parsed.push(call); - remaining = remainder.trim_start(); - } - } - } - - Ok(parsed) -} - -#[cfg(test)] -mod tests { - use serde_json::json; - - use llama_cpp_bindings_types::bracketed_json_shape::BracketedJsonShape; - use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; - use llama_cpp_bindings_types::tool_call_args_shape::ToolCallArgsShape; - use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; - use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; - - use super::parse; - use crate::error::bracketed_args_failure::BracketedArgsFailure; - - fn mistral3_markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "[TOOL_CALLS]".to_owned(), - close: String::new(), - args_shape: ToolCallArgsShape::BracketedJson(BracketedJsonShape { - name_args_separator: "[ARGS]".to_owned(), - }), - } - } - - fn mistral3_shape() -> BracketedJsonShape { - BracketedJsonShape { - name_args_separator: "[ARGS]".to_owned(), - } - } - - #[test] - fn parses_single_tool_call_with_open_marker_present() { - let parsed = parse( - "[TOOL_CALLS]get_weather[ARGS]{\"location\":\"Paris\"}", - &mistral3_markers(), - &mistral3_shape(), - ) - .expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "get_weather"); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ); - } - - #[test] - fn parses_single_tool_call_when_classifier_stripped_open_marker() { - let parsed = parse( - "get_weather[ARGS]{\"location\":\"Paris\"}", - &mistral3_markers(), - &mistral3_shape(), - ) - .expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "get_weather"); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ); - } - - #[test] - fn parses_two_consecutive_tool_calls_with_repeated_open_marker() { - let parsed = parse( - "[TOOL_CALLS]a[ARGS]{\"x\":1}[TOOL_CALLS]b[ARGS]{\"y\":2}", - &mistral3_markers(), - &mistral3_shape(), - ) - .expect("must parse"); - - assert_eq!(parsed.len(), 2); - assert_eq!(parsed[0].name, "a"); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"x": 1})) - ); - assert_eq!(parsed[1].name, "b"); - assert_eq!( - parsed[1].arguments, - ToolCallArguments::ValidJson(json!({"y": 2})) - ); - } - - #[test] - fn malformed_json_arguments_are_reported_with_the_parser_message() { - assert_eq!( - parse( - "[TOOL_CALLS]get_weather[ARGS]{\"location\":}", - &mistral3_markers(), - &mistral3_shape(), - ), - Err(BracketedArgsFailure::InvalidJsonArguments { - tool_name: "get_weather".to_owned(), - message: "expected value at line 1 column 13".to_owned(), - }) - ); - } - - #[test] - fn whitespace_only_arguments_are_reported_as_unterminated() { - let failure = parse( - "[TOOL_CALLS]get_weather[ARGS] ", - &mistral3_markers(), - &mistral3_shape(), - ) - .expect_err("truncated arguments must produce a typed failure"); - - assert_eq!( - failure, - BracketedArgsFailure::UnterminatedArguments { - tool_name: "get_weather".to_owned(), - }, - ); - } - - #[test] - fn returns_empty_vec_for_separator_with_only_whitespace_name() { - let parsed = parse( - "[TOOL_CALLS] [ARGS]{\"x\":1}", - &mistral3_markers(), - &mistral3_shape(), - ) - .expect("whitespace-name input must parse"); - - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn returns_empty_vec_when_shape_has_empty_separator() { - let mut shape = mistral3_shape(); - shape.name_args_separator.clear(); - let parsed = parse( - "[TOOL_CALLS]get_weather[ARGS]{\"x\":1}", - &mistral3_markers(), - &shape, - ) - .expect("empty-separator shape must parse"); - - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn returns_empty_vec_for_empty_body() { - let parsed = - parse("", &mistral3_markers(), &mistral3_shape()).expect("empty body must parse"); - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn returns_empty_vec_when_body_lacks_separator() { - let parsed = parse( - "plain text without separator", - &mistral3_markers(), - &mistral3_shape(), - ) - .expect("body without separator must parse"); - assert_eq!(parsed, Vec::::new()); - } -} diff --git a/llama-cpp-bindings/src/tool_call_format/consume_optional_prefix.rs b/llama-cpp-bindings/src/tool_call_format/consume_optional_prefix.rs deleted file mode 100644 index bc1ea817b..000000000 --- a/llama-cpp-bindings/src/tool_call_format/consume_optional_prefix.rs +++ /dev/null @@ -1,4 +0,0 @@ -#[must_use] -pub fn consume_optional_prefix<'body>(input: &'body str, literal: &str) -> &'body str { - input.strip_prefix(literal).unwrap_or(input) -} diff --git a/llama-cpp-bindings/src/tool_call_format/json_object.rs b/llama-cpp-bindings/src/tool_call_format/json_object.rs deleted file mode 100644 index a05193831..000000000 --- a/llama-cpp-bindings/src/tool_call_format/json_object.rs +++ /dev/null @@ -1,197 +0,0 @@ -use llama_cpp_bindings_types::json_object_shape::JsonObjectShape; -use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; -use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; -use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; - -use crate::error::json_object_failure::JsonObjectFailure; - -fn try_parse_one_object( - input: &str, - shape: &JsonObjectShape, -) -> Result, JsonObjectFailure> { - let start = input.find('{').unwrap_or(input.len()); - let mut stream = serde_json::Deserializer::from_str(&input[start..]) - .into_iter::>(); - let map = match stream.next() { - Some(Ok(map)) => map, - Some(Err(err)) => { - return Err(JsonObjectFailure::InvalidJson { - message: err.to_string(), - }); - } - None => return Ok(None), - }; - let consumed = stream.byte_offset(); - - let Some(name_value) = map.get(&shape.name_field) else { - return Ok(None); - }; - let serde_json::Value::String(name) = name_value else { - return Ok(None); - }; - - let arguments_value = map - .get(&shape.arguments_field) - .cloned() - .unwrap_or(serde_json::Value::Object(serde_json::Map::new())); - let arguments = ToolCallArguments::from_string(arguments_value.to_string()); - - let trailing_extras = map - .keys() - .any(|key| key != &shape.name_field && key != &shape.arguments_field); - if trailing_extras { - return Ok(None); - } - - Ok(Some(( - ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - name.clone(), - arguments, - ), - start + consumed, - ))) -} - -/// # Errors -/// -/// Returns [`JsonObjectFailure`] when the body contains a JSON object that -/// looks like a tool call (matches the open brace at start) but the JSON itself -/// is malformed. -pub fn parse( - body: &str, - shape: &JsonObjectShape, -) -> Result, JsonObjectFailure> { - if shape.name_field.is_empty() || shape.arguments_field.is_empty() { - return Ok(Vec::new()); - } - - let mut parsed = Vec::new(); - let mut remaining = body; - - while let Some((call, consumed)) = try_parse_one_object(remaining, shape)? { - parsed.push(call); - remaining = &remaining[consumed..]; - } - - Ok(parsed) -} - -#[cfg(test)] -mod tests { - use serde_json::json; - - use llama_cpp_bindings_types::json_object_shape::JsonObjectShape; - use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; - use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; - - use super::parse; - use crate::error::json_object_failure::JsonObjectFailure; - - fn qwen3_shape() -> JsonObjectShape { - JsonObjectShape { - name_field: "name".to_owned(), - arguments_field: "arguments".to_owned(), - } - } - - #[test] - fn parses_single_json_object_with_name_and_arguments() { - let parsed = parse( - r#"{"name": "get_weather", "arguments": {"location": "Paris"}}"#, - &qwen3_shape(), - ) - .expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "get_weather"); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ); - } - - #[test] - fn parses_json_object_after_leading_whitespace_and_newlines() { - let parsed = parse( - "\n {\"name\": \"f\", \"arguments\": {\"a\": 1}}\n", - &qwen3_shape(), - ) - .expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "f"); - } - - #[test] - fn parses_two_consecutive_json_objects() { - let parsed = parse( - r#"{"name": "a", "arguments": {}}{"name": "b", "arguments": {"x": 2}}"#, - &qwen3_shape(), - ) - .expect("must parse"); - - assert_eq!(parsed.len(), 2); - assert_eq!(parsed[0].name, "a"); - assert_eq!(parsed[1].name, "b"); - } - - #[test] - fn parses_object_with_arguments_field_missing_yields_empty_arguments() { - let parsed = parse(r#"{"name": "ping"}"#, &qwen3_shape()).expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "ping"); - assert_eq!(parsed[0].arguments, ToolCallArguments::ValidJson(json!({})),); - } - - #[test] - fn rejects_json_object_with_extra_unexpected_top_level_keys() { - let parsed = parse( - r#"{"name": "f", "arguments": {}, "extra": 1}"#, - &qwen3_shape(), - ) - .expect("must parse"); - - assert!(parsed.is_empty(), "extra top-level key must reject"); - } - - #[test] - fn rejects_json_object_with_non_string_name() { - let parsed = - parse(r#"{"name": 123, "arguments": {}}"#, &qwen3_shape()).expect("must parse"); - - assert!(parsed.is_empty(), "non-string name must reject"); - } - - #[test] - fn input_without_an_open_brace_parses_no_tool_calls() { - let parsed = parse("plain content", &qwen3_shape()).expect("must parse"); - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn malformed_json_is_reported_with_the_parser_message() { - assert_eq!( - parse(r#"{"name": "f", "arguments": {"a": }"#, &qwen3_shape()), - Err(JsonObjectFailure::InvalidJson { - message: "expected value at line 1 column 34".to_owned(), - }) - ); - } - - #[test] - fn returns_empty_when_object_is_not_a_tool_call_shape() { - let parsed = parse("{ \"foo\": 1 }", &qwen3_shape()).expect("must parse"); - - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn returns_empty_when_shape_has_empty_required_field() { - let mut shape = qwen3_shape(); - shape.name_field.clear(); - let parsed = parse(r#"{"name": "x", "arguments": {}}"#, &shape).expect("must parse"); - assert_eq!(parsed, Vec::::new()); - } -} diff --git a/llama-cpp-bindings/src/tool_call_format/key_value_xml_tags.rs b/llama-cpp-bindings/src/tool_call_format/key_value_xml_tags.rs deleted file mode 100644 index dedae7f91..000000000 --- a/llama-cpp-bindings/src/tool_call_format/key_value_xml_tags.rs +++ /dev/null @@ -1,339 +0,0 @@ -use nom::IResult; -use nom::Parser; -use nom::bytes::complete::take_until; - -use llama_cpp_bindings_types::key_value_xml_tags_shape::KeyValueXmlTagsShape; -use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; -use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; -use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; -use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; - -use crate::error::key_value_xml_tags_failure::KeyValueXmlTagsFailure; -use crate::tool_call_format::parse_step::ParseStep; -use crate::tool_call_format::scalar_value_to_json::scalar_value_to_json; - -const fn shape_is_complete(shape: &KeyValueXmlTagsShape) -> bool { - !shape.key_open.is_empty() - && !shape.key_close.is_empty() - && !shape.value_open.is_empty() - && !shape.value_close.is_empty() -} - -fn skip_to_next_open<'body>(input: &'body str, open: &str) -> Option<&'body str> { - let take_result: IResult<&'body str, &'body str> = take_until(open).parse(input); - let (after_open_inclusive, _) = take_result.ok()?; - - Some(&after_open_inclusive[open.len()..]) -} - -fn parse_one_parameter<'body>( - input: &'body str, - shape: &KeyValueXmlTagsShape, - function_name: &str, -) -> Result, KeyValueXmlTagsFailure> { - let take_result: IResult<&'body str, &'body str> = - take_until(shape.key_open.as_str()).parse(input); - let Ok((after_key_open_inclusive, _)) = take_result else { - return Ok(None); - }; - let after_key_open = &after_key_open_inclusive[shape.key_open.len()..]; - - let key_close_position = after_key_open - .find(shape.key_close.as_str()) - .ok_or_else(|| KeyValueXmlTagsFailure::UnclosedKeyTag { - function_name: function_name.to_owned(), - expected_close: shape.key_close.clone(), - })?; - let key = after_key_open[..key_close_position].trim().to_owned(); - if key.is_empty() { - return Err(KeyValueXmlTagsFailure::EmptyKey { - function_name: function_name.to_owned(), - }); - } - let after_key_close = &after_key_open[key_close_position + shape.key_close.len()..]; - - let value_open_take: IResult<&str, &str> = - take_until(shape.value_open.as_str()).parse(after_key_close); - let Ok((after_value_open_inclusive, _)) = value_open_take else { - return Err(KeyValueXmlTagsFailure::MissingValueTag { - function_name: function_name.to_owned(), - key, - expected_open: shape.value_open.clone(), - }); - }; - let after_value_open = &after_value_open_inclusive[shape.value_open.len()..]; - - let value_close_position = after_value_open - .find(shape.value_close.as_str()) - .ok_or_else(|| KeyValueXmlTagsFailure::UnclosedValueTag { - function_name: function_name.to_owned(), - key: key.clone(), - expected_close: shape.value_close.clone(), - })?; - let raw_value = &after_value_open[..value_close_position]; - let value = scalar_value_to_json(raw_value); - let after_value_close = &after_value_open[value_close_position + shape.value_close.len()..]; - - Ok(Some((key, value, after_value_close))) -} - -fn collect_parameters( - function_body: &str, - shape: &KeyValueXmlTagsShape, - function_name: &str, -) -> Result, KeyValueXmlTagsFailure> { - let mut parameters = serde_json::Map::new(); - let mut remaining = function_body; - - while let Some((key, value, rest)) = parse_one_parameter(remaining, shape, function_name)? { - parameters.insert(key, value); - remaining = rest; - } - - Ok(parameters) -} - -fn parse_one_call<'body>( - input: &'body str, - markers: &ToolCallMarkers, - shape: &KeyValueXmlTagsShape, -) -> Result, KeyValueXmlTagsFailure> { - let Some(after_open) = skip_to_next_open(input, &markers.open) else { - return Ok(ParseStep::Done); - }; - - let Some(close_position) = after_open.find(markers.close.as_str()) else { - return Err(KeyValueXmlTagsFailure::UnclosedFunctionBlock { - expected_close: markers.close.clone(), - }); - }; - let function_block = &after_open[..close_position]; - let after_function_close = &after_open[close_position + markers.close.len()..]; - - let (name_end, has_args) = function_block - .find(shape.key_open.as_str()) - .map_or((function_block.len(), false), |position| (position, true)); - let function_name = function_block[..name_end].trim().to_owned(); - if function_name.is_empty() { - return Err(KeyValueXmlTagsFailure::EmptyFunctionName); - } - - let args_section = if has_args { - &function_block[name_end..] - } else { - "" - }; - let arguments_object = collect_parameters(args_section, shape, &function_name)?; - let arguments_value = serde_json::Value::Object(arguments_object); - let arguments = ToolCallArguments::from_string(arguments_value.to_string()); - - Ok(ParseStep::Call { - call: ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - function_name, - arguments, - ), - remainder: after_function_close, - }) -} - -/// # Errors -/// -/// Returns [`KeyValueXmlTagsFailure`] when the body looks like a key-value-XML -/// tool-call block (matches the open marker) but contains a structural issue: -/// empty function/key name, missing key/value tag, or unclosed function block. -pub fn parse( - body: &str, - markers: &ToolCallMarkers, - shape: &KeyValueXmlTagsShape, -) -> Result, KeyValueXmlTagsFailure> { - if !shape_is_complete(shape) || markers.open.is_empty() || markers.close.is_empty() { - return Ok(Vec::new()); - } - - let mut parsed = Vec::new(); - let mut remaining = body; - - loop { - match parse_one_call(remaining, markers, shape)? { - ParseStep::Done => break, - ParseStep::Call { call, remainder } => { - parsed.push(call); - remaining = remainder; - } - } - } - - Ok(parsed) -} - -#[cfg(test)] -mod tests { - use serde_json::json; - - use llama_cpp_bindings_types::key_value_xml_tags_shape::KeyValueXmlTagsShape; - use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; - use llama_cpp_bindings_types::tool_call_args_shape::ToolCallArgsShape; - use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; - use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; - - use super::parse; - use crate::error::key_value_xml_tags_failure::KeyValueXmlTagsFailure; - - fn glm47_markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "".to_owned(), - close: "".to_owned(), - args_shape: ToolCallArgsShape::KeyValueXmlTags(glm47_shape()), - } - } - - fn glm47_shape() -> KeyValueXmlTagsShape { - KeyValueXmlTagsShape { - key_open: "".to_owned(), - key_close: "".to_owned(), - value_open: "".to_owned(), - value_close: "".to_owned(), - } - } - - #[test] - fn parses_single_call_with_one_argument() { - let body = "get_weatherlocationParis"; - let parsed = parse(body, &glm47_markers(), &glm47_shape()).expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "get_weather"); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ); - } - - #[test] - fn parses_call_with_multiple_arguments() { - let body = "set_thermostatroomkitchencelsius21"; - let parsed = parse(body, &glm47_markers(), &glm47_shape()).expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "set_thermostat"); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"room": "kitchen", "celsius": 21})), - ); - } - - #[test] - fn parses_two_calls_in_one_body() { - let body = "ax1by2"; - let parsed = parse(body, &glm47_markers(), &glm47_shape()).expect("must parse"); - - assert_eq!(parsed.len(), 2); - assert_eq!(parsed[0].name, "a"); - assert_eq!(parsed[1].name, "b"); - } - - #[test] - fn parses_call_with_no_arguments() { - let body = "ping"; - let parsed = parse(body, &glm47_markers(), &glm47_shape()).expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "ping"); - assert_eq!(parsed[0].arguments, ToolCallArguments::ValidJson(json!({})),); - } - - #[test] - fn rejects_unclosed_function_block_with_typed_failure() { - let body = "get_weatherlocationParis"; - let result = parse(body, &glm47_markers(), &glm47_shape()); - - assert_eq!( - result, - Err(KeyValueXmlTagsFailure::UnclosedFunctionBlock { - expected_close: "".to_owned(), - }), - ); - } - - #[test] - fn rejects_empty_function_name_with_typed_failure() { - let body = "kv"; - let result = parse(body, &glm47_markers(), &glm47_shape()); - - assert_eq!(result, Err(KeyValueXmlTagsFailure::EmptyFunctionName)); - } - - #[test] - fn rejects_unclosed_key_tag_with_typed_failure() { - let body = "flocation"; - let result = parse(body, &glm47_markers(), &glm47_shape()); - - assert_eq!( - result, - Err(KeyValueXmlTagsFailure::UnclosedKeyTag { - function_name: "f".to_owned(), - expected_close: "".to_owned(), - }), - ); - } - - #[test] - fn rejects_missing_value_tag_with_typed_failure() { - let body = "flocationParis"; - let result = parse(body, &glm47_markers(), &glm47_shape()); - - assert_eq!( - result, - Err(KeyValueXmlTagsFailure::MissingValueTag { - function_name: "f".to_owned(), - key: "location".to_owned(), - expected_open: "".to_owned(), - }), - ); - } - - #[test] - fn rejects_empty_key_with_typed_failure() { - let body = "fParis"; - let result = parse(body, &glm47_markers(), &glm47_shape()); - - assert_eq!( - result, - Err(KeyValueXmlTagsFailure::EmptyKey { - function_name: "f".to_owned(), - }), - ); - } - - #[test] - fn rejects_unclosed_value_tag_with_typed_failure() { - let body = "flocationParis"; - let result = parse(body, &glm47_markers(), &glm47_shape()); - - assert_eq!( - result, - Err(KeyValueXmlTagsFailure::UnclosedValueTag { - function_name: "f".to_owned(), - key: "location".to_owned(), - expected_close: "".to_owned(), - }), - ); - } - - #[test] - fn returns_empty_for_body_without_open_marker() { - let parsed = - parse("plain text", &glm47_markers(), &glm47_shape()).expect("must parse empty"); - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn returns_empty_when_shape_is_incomplete() { - let mut shape = glm47_shape(); - shape.value_close.clear(); - let body = "fkv"; - let parsed = parse(body, &glm47_markers(), &shape).expect("must parse empty"); - assert_eq!(parsed, Vec::::new()); - } -} diff --git a/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs b/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs deleted file mode 100644 index 27d5765ce..000000000 --- a/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs +++ /dev/null @@ -1,515 +0,0 @@ -use llama_cpp_bindings_types::paired_quote_shape::PairedQuoteShape; -use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; -use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; -use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; -use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; -use llama_cpp_bindings_types::tool_call_value_quote::ToolCallValueQuote; - -use crate::error::paired_quote_failure::PairedQuoteFailure; -use crate::tool_call_format::consume_optional_prefix::consume_optional_prefix; -use crate::tool_call_format::parse_step::ParseStep; -use crate::tool_call_format::parsed_prefix::ParsedPrefix; -use crate::tool_call_format::scalar_value_to_json::scalar_value_to_json; -use crate::tool_call_format::separator_split::SeparatorSplit; - -fn bare_value_to_json(text: &str) -> serde_json::Value { - if text.is_empty() { - return serde_json::Value::Null; - } - - scalar_value_to_json(text) -} - -fn find_bare_value_end(input: &str, close_marker: &str) -> usize { - for (byte_index, character) in input.char_indices() { - if character == ',' { - return byte_index; - } - if !close_marker.is_empty() && input[byte_index..].starts_with(close_marker) { - return byte_index; - } - } - - input.len() -} - -fn parse_one_key<'body>( - input: &'body str, - tool_name: &str, -) -> Result, PairedQuoteFailure> { - let Some((key_raw, after_colon)) = input.split_once(':') else { - return Err(PairedQuoteFailure::UnclosedArgumentBlock { - tool_name: tool_name.to_owned(), - state: "key", - }); - }; - let key = key_raw.trim().to_owned(); - if key.is_empty() { - return Err(PairedQuoteFailure::EmptyKey { - tool_name: tool_name.to_owned(), - }); - } - - Ok(ParsedPrefix { - value: key, - remaining: after_colon, - }) -} - -fn parse_one_value<'body>( - input: &'body str, - value_quote: &ToolCallValueQuote, - close_marker: &str, - tool_name: &str, - key: &str, -) -> Result, PairedQuoteFailure> { - let trimmed = input.trim_start(); - - if !value_quote.open.is_empty() - && !value_quote.close.is_empty() - && let Some(after_open) = trimmed.strip_prefix(value_quote.open.as_str()) - { - let Some(close_position) = after_open.find(value_quote.close.as_str()) else { - return Err(PairedQuoteFailure::UnclosedQuotedValue { - tool_name: tool_name.to_owned(), - key: key.to_owned(), - }); - }; - let value_text = after_open[..close_position].to_owned(); - let after_close = &after_open[close_position + value_quote.close.len()..]; - - return Ok(ParsedPrefix { - value: serde_json::Value::String(value_text), - remaining: after_close, - }); - } - - let bare_end = find_bare_value_end(trimmed, close_marker); - let bare_text = trimmed[..bare_end].trim(); - let value = bare_value_to_json(bare_text); - - Ok(ParsedPrefix { - value, - remaining: &trimmed[bare_end..], - }) -} - -fn parse_args_body<'body>( - input: &'body str, - value_quote: &ToolCallValueQuote, - close_marker: &str, - tool_name: &str, -) -> Result>, PairedQuoteFailure> { - let mut map = serde_json::Map::new(); - let mut remaining = input.trim_start(); - - loop { - if remaining.is_empty() { - return Ok(ParsedPrefix { - value: map, - remaining, - }); - } - if !close_marker.is_empty() - && let Some(after_close) = remaining.strip_prefix(close_marker) - { - return Ok(ParsedPrefix { - value: map, - remaining: after_close, - }); - } - - let ParsedPrefix { - value: key, - remaining: after_key, - } = parse_one_key(remaining, tool_name)?; - let ParsedPrefix { - value, - remaining: after_value, - } = parse_one_value(after_key, value_quote, close_marker, tool_name, &key)?; - map.insert(key.clone(), value); - - remaining = after_value.trim_start(); - if !close_marker.is_empty() - && let Some(after_close) = remaining.strip_prefix(close_marker) - { - return Ok(ParsedPrefix { - value: map, - remaining: after_close, - }); - } - if let Some(after_comma) = remaining.strip_prefix(',') { - remaining = after_comma.trim_start(); - continue; - } - - let Some(character) = remaining.chars().next() else { - return Ok(ParsedPrefix { - value: map, - remaining, - }); - }; - - return Err(PairedQuoteFailure::UnexpectedCharAfterValue { - tool_name: tool_name.to_owned(), - key, - character, - }); - } -} - -fn parse_one_call<'body>( - input: &'body str, - markers: &ToolCallMarkers, - shape: &PairedQuoteShape, -) -> Result, PairedQuoteFailure> { - if input.is_empty() { - return Ok(ParseStep::Done); - } - - let after_open = consume_optional_prefix(input, markers.open.as_str()); - - let Some(SeparatorSplit { - before: name_raw, - after: after_separator, - }) = SeparatorSplit::at_first(after_open, shape.name_args_separator.as_str()) - else { - return Ok(ParseStep::Done); - }; - - let name = name_raw.trim().to_owned(); - if name.is_empty() { - return Ok(ParseStep::Done); - } - - let ParsedPrefix { - value: args_object, - remaining: after_args, - } = parse_args_body( - after_separator, - &shape.value_quote, - markers.close.as_str(), - &name, - )?; - let arguments_value = serde_json::Value::Object(args_object); - - Ok(ParseStep::Call { - call: ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - name, - ToolCallArguments::ValidJson(arguments_value), - ), - remainder: after_args, - }) -} - -/// # Errors -/// -/// Returns [`PairedQuoteFailure`] when the body looks like a paired-quote -/// tool-call block (matches the open marker and separator) but contains a -/// structural issue: empty key, unclosed quoted value, unexpected character -/// after a value, or an unfinished argument block. -pub fn parse( - body: &str, - markers: &ToolCallMarkers, - shape: &PairedQuoteShape, -) -> Result, PairedQuoteFailure> { - if shape.name_args_separator.is_empty() { - return Ok(Vec::new()); - } - - let mut parsed = Vec::new(); - let mut remaining = body.trim_start(); - - loop { - match parse_one_call(remaining, markers, shape)? { - ParseStep::Done => break, - ParseStep::Call { call, remainder } => { - parsed.push(call); - remaining = remainder.trim_start(); - } - } - } - - Ok(parsed) -} - -#[cfg(test)] -mod tests { - use serde_json::json; - - use llama_cpp_bindings_types::paired_quote_shape::PairedQuoteShape; - use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; - use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; - use llama_cpp_bindings_types::tool_call_args_shape::ToolCallArgsShape; - use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; - use llama_cpp_bindings_types::tool_call_markers::ToolCallMarkers; - use llama_cpp_bindings_types::tool_call_value_quote::ToolCallValueQuote; - - use super::parse; - use crate::error::paired_quote_failure::PairedQuoteFailure; - - fn gemma4_markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "<|tool_call>call:".to_owned(), - close: "}".to_owned(), - args_shape: ToolCallArgsShape::PairedQuote(gemma4_shape()), - } - } - - fn gemma4_shape() -> PairedQuoteShape { - PairedQuoteShape { - name_args_separator: "{".to_owned(), - value_quote: ToolCallValueQuote { - open: "<|\"|>".to_owned(), - close: "<|\"|>".to_owned(), - }, - } - } - - fn gemma4_call(arguments_body: &str) -> String { - let mut call = String::from("<|tool_call>call:f"); - call.push('{'); - call.push_str(arguments_body); - call.push('}'); - - call - } - - #[test] - fn parses_single_quoted_string_argument_with_full_markers() { - let parsed = parse( - "<|tool_call>call:get_weather{location:<|\"|>Paris<|\"|>}", - &gemma4_markers(), - &gemma4_shape(), - ) - .expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "get_weather"); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ); - } - - #[test] - fn parses_classifier_stripped_body_without_open_or_close() { - let parsed = parse( - "get_weather{location:<|\"|>Paris<|\"|>", - &gemma4_markers(), - &gemma4_shape(), - ) - .expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "get_weather"); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ); - } - - #[test] - fn parses_multiple_quoted_string_arguments() { - let parsed = parse( - "<|tool_call>call:f{a:<|\"|>1<|\"|>,b:<|\"|>2<|\"|>}", - &gemma4_markers(), - &gemma4_shape(), - ) - .expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"a": "1", "b": "2"})), - ); - } - - #[test] - fn parses_comma_separated_bare_values() { - let parsed = parse( - &gemma4_call("a:42,b:true"), - &gemma4_markers(), - &gemma4_shape(), - ); - - assert_eq!( - parsed, - Ok(vec![ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - "f".to_owned(), - ToolCallArguments::ValidJson(json!({"a": 42, "b": true})), - )]), - ); - } - - #[test] - fn rejects_unclosed_quoted_value_with_typed_failure() { - let result = parse( - "<|tool_call>call:f{a:<|\"|>oops", - &gemma4_markers(), - &gemma4_shape(), - ); - - assert_eq!( - result.expect_err("unclosed quote must produce a typed failure"), - PairedQuoteFailure::UnclosedQuotedValue { - tool_name: "f".to_owned(), - key: "a".to_owned(), - }, - ); - } - - #[test] - fn rejects_unexpected_char_after_value_with_typed_failure() { - let result = parse( - "<|tool_call>call:f{a:<|\"|>v<|\"|>$bad}", - &gemma4_markers(), - &gemma4_shape(), - ); - - assert_eq!( - result.expect_err("garbage after value must produce a typed failure"), - PairedQuoteFailure::UnexpectedCharAfterValue { - tool_name: "f".to_owned(), - key: "a".to_owned(), - character: '$', - }, - ); - } - - #[test] - fn returns_empty_vec_for_empty_body() { - let parsed = parse("", &gemma4_markers(), &gemma4_shape()).expect("empty body must parse"); - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn returns_empty_vec_when_body_lacks_separator() { - let parsed = parse("no separator anywhere", &gemma4_markers(), &gemma4_shape()) - .expect("body without separator must parse"); - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn parses_args_body_terminated_by_end_of_input_after_quoted_value() { - let parsed = parse( - "<|tool_call>call:f{x:<|\"|>v<|\"|>", - &gemma4_markers(), - &gemma4_shape(), - ) - .expect("end-of-input after quoted value must parse"); - - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"x": "v"})), - ); - } - - #[test] - fn parses_args_body_terminated_by_end_of_input_after_bare_value() { - let parsed = parse( - "<|tool_call>call:f{n:42", - &gemma4_markers(), - &gemma4_shape(), - ) - .expect("end-of-input after bare value must parse"); - - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"n": 42})), - ); - } - - #[test] - fn rejects_empty_key_with_typed_failure() { - let result = parse(&gemma4_call(":42"), &gemma4_markers(), &gemma4_shape()); - - assert_eq!( - result.expect_err("empty key must produce a typed failure"), - PairedQuoteFailure::EmptyKey { - tool_name: "f".to_owned(), - }, - ); - } - - #[test] - fn rejects_args_body_without_key_colon_with_typed_failure() { - let result = parse( - "<|tool_call>call:f{noColonHere", - &gemma4_markers(), - &gemma4_shape(), - ); - - assert_eq!( - result.expect_err("args body without colon must produce a typed failure"), - PairedQuoteFailure::UnclosedArgumentBlock { - tool_name: "f".to_owned(), - state: "key", - }, - ); - } - - #[test] - fn parses_empty_bare_value_as_null() { - let parsed = parse(&gemma4_call("a:"), &gemma4_markers(), &gemma4_shape()) - .expect("empty bare value must parse"); - - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"a": null})), - ); - } - - #[test] - fn parses_call_with_empty_args_body_terminated_by_end_of_input() { - let parsed = parse("<|tool_call>call:f{", &gemma4_markers(), &gemma4_shape()) - .expect("empty args body must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "f"); - assert_eq!(parsed[0].arguments, ToolCallArguments::ValidJson(json!({})),); - } - - #[test] - fn parses_call_with_empty_args_body_closed_by_marker() { - let parsed = parse("<|tool_call>call:f{}", &gemma4_markers(), &gemma4_shape()) - .expect("empty args body closed by marker must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "f"); - assert_eq!(parsed[0].arguments, ToolCallArguments::ValidJson(json!({})),); - } - - #[test] - fn stops_parsing_when_tool_name_is_empty() { - let parsed = parse( - "<|tool_call>call:{a:<|\"|>v<|\"|>}", - &gemma4_markers(), - &gemma4_shape(), - ) - .expect("empty tool name must yield no calls"); - - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn returns_empty_vec_when_separator_is_empty() { - let shape = PairedQuoteShape { - name_args_separator: String::new(), - value_quote: ToolCallValueQuote { - open: "<|\"|>".to_owned(), - close: "<|\"|>".to_owned(), - }, - }; - let parsed = parse( - "<|tool_call>call:f{a:<|\"|>v<|\"|>}", - &gemma4_markers(), - &shape, - ) - .expect("empty separator must yield no calls"); - - assert_eq!(parsed, Vec::::new()); - } -} diff --git a/llama-cpp-bindings/src/tool_call_format/parse_step.rs b/llama-cpp-bindings/src/tool_call_format/parse_step.rs deleted file mode 100644 index f2005932f..000000000 --- a/llama-cpp-bindings/src/tool_call_format/parse_step.rs +++ /dev/null @@ -1,10 +0,0 @@ -use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; - -#[derive(Debug)] -pub enum ParseStep<'body> { - Done, - Call { - call: ParsedToolCall, - remainder: &'body str, - }, -} diff --git a/llama-cpp-bindings/src/tool_call_format/parsed_prefix.rs b/llama-cpp-bindings/src/tool_call_format/parsed_prefix.rs deleted file mode 100644 index e29c60581..000000000 --- a/llama-cpp-bindings/src/tool_call_format/parsed_prefix.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[derive(Debug, Eq, PartialEq)] -pub struct ParsedPrefix<'body, TValue> { - pub value: TValue, - pub remaining: &'body str, -} diff --git a/llama-cpp-bindings/src/tool_call_format/scalar_value_to_json.rs b/llama-cpp-bindings/src/tool_call_format/scalar_value_to_json.rs deleted file mode 100644 index 6efe2fc98..000000000 --- a/llama-cpp-bindings/src/tool_call_format/scalar_value_to_json.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[must_use] -pub fn scalar_value_to_json(raw: &str) -> serde_json::Value { - serde_json::from_str::(raw) - .unwrap_or_else(|_not_json| serde_json::Value::String(raw.to_owned())) -} diff --git a/llama-cpp-bindings/src/tool_call_format/separator_split.rs b/llama-cpp-bindings/src/tool_call_format/separator_split.rs deleted file mode 100644 index 3f9eca60b..000000000 --- a/llama-cpp-bindings/src/tool_call_format/separator_split.rs +++ /dev/null @@ -1,14 +0,0 @@ -#[derive(Debug, Eq, PartialEq)] -pub struct SeparatorSplit<'body> { - pub before: &'body str, - pub after: &'body str, -} - -impl<'body> SeparatorSplit<'body> { - #[must_use] - pub fn at_first(input: &'body str, separator: &str) -> Option { - input - .split_once(separator) - .map(|(before, after)| Self { before, after }) - } -} diff --git a/llama-cpp-bindings/src/tool_call_format/tool_call_format_outcome.rs b/llama-cpp-bindings/src/tool_call_format/tool_call_format_outcome.rs deleted file mode 100644 index 17f9ab868..000000000 --- a/llama-cpp-bindings/src/tool_call_format/tool_call_format_outcome.rs +++ /dev/null @@ -1,10 +0,0 @@ -use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; - -use crate::error::tool_call_format_failure::ToolCallFormatFailure; - -#[derive(Debug, Eq, PartialEq)] -pub enum ToolCallFormatOutcome { - Parsed(Vec), - NoMatch, - Failed(ToolCallFormatFailure), -} diff --git a/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs b/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs deleted file mode 100644 index 88db0f5d4..000000000 --- a/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs +++ /dev/null @@ -1,363 +0,0 @@ -use nom::IResult; -use nom::Parser; -use nom::bytes::complete::take_until; - -use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; -use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; -use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; -use llama_cpp_bindings_types::xml_tags_shape::XmlTagsShape; - -use crate::error::xml_function_tags_failure::XmlFunctionTagsFailure; -use crate::tool_call_format::scalar_value_to_json::scalar_value_to_json; - -const fn shape_is_complete(shape: &XmlTagsShape) -> bool { - !shape.function_open_prefix.is_empty() - && !shape.function_close.is_empty() - && !shape.parameter_open_prefix.is_empty() - && !shape.parameter_close.is_empty() -} - -fn trim_surrounding_newlines(input: &str) -> &str { - input.trim_start_matches('\n').trim_end_matches('\n') -} - -fn locate_tag_name_end(after_prefix: &str) -> Option { - let close_position = after_prefix.find('>'); - let next_open_position = after_prefix.find('<'); - - match (close_position, next_open_position) { - (Some(close), Some(open)) if open < close => None, - (Some(close), _) => Some(close), - (None, _) => None, - } -} - -fn skip_to_next_function_open<'body>( - input: &'body str, - function_open_prefix: &str, -) -> Option<&'body str> { - let take_result: IResult<&'body str, &'body str> = - take_until(function_open_prefix).parse(input); - let (after_prefix_inclusive, _) = take_result.ok()?; - - Some(&after_prefix_inclusive[function_open_prefix.len()..]) -} - -fn parse_one_parameter<'body>( - input: &'body str, - shape: &XmlTagsShape, - function_name: &str, -) -> Result, XmlFunctionTagsFailure> { - let take_result: IResult<&'body str, &'body str> = - take_until(shape.parameter_open_prefix.as_str()).parse(input); - let Ok((after_prefix_inclusive, _)) = take_result else { - return Ok(None); - }; - let after_prefix = &after_prefix_inclusive[shape.parameter_open_prefix.len()..]; - - let Some(name_end) = locate_tag_name_end(after_prefix) else { - return Err(XmlFunctionTagsFailure::UnclosedParameterBlock { - function_name: function_name.to_owned(), - parameter_name: String::new(), - expected_close: shape.parameter_close.clone(), - }); - }; - let parameter_name = after_prefix[..name_end].trim().to_owned(); - if parameter_name.is_empty() { - return Err(XmlFunctionTagsFailure::EmptyParameterName { - function_name: function_name.to_owned(), - }); - } - let value_start = &after_prefix[name_end + 1..]; - - let Some(value_end_position) = value_start.find(shape.parameter_close.as_str()) else { - return Err(XmlFunctionTagsFailure::UnclosedParameterBlock { - function_name: function_name.to_owned(), - parameter_name, - expected_close: shape.parameter_close.clone(), - }); - }; - let raw_value = trim_surrounding_newlines(&value_start[..value_end_position]); - let after_close = &value_start[value_end_position + shape.parameter_close.len()..]; - let parameter_value = scalar_value_to_json(raw_value); - - Ok(Some((parameter_name, parameter_value, after_close))) -} - -fn collect_parameters( - function_body: &str, - shape: &XmlTagsShape, - function_name: &str, -) -> Result, XmlFunctionTagsFailure> { - let mut parameters = serde_json::Map::new(); - let mut remaining = function_body; - - while let Some((parameter_name, parameter_value, rest)) = - parse_one_parameter(remaining, shape, function_name)? - { - parameters.insert(parameter_name, parameter_value); - remaining = rest; - } - - Ok(parameters) -} - -fn parse_one_function<'body>( - input: &'body str, - shape: &XmlTagsShape, -) -> Result, XmlFunctionTagsFailure> { - let Some(after_function_prefix) = - skip_to_next_function_open(input, &shape.function_open_prefix) - else { - return Ok(None); - }; - - let Some(name_end) = locate_tag_name_end(after_function_prefix) else { - return Err(XmlFunctionTagsFailure::UnclosedFunctionBlock { - function_name: String::new(), - expected_close: shape.function_close.clone(), - }); - }; - let function_name = after_function_prefix[..name_end].trim().to_owned(); - if function_name.is_empty() { - return Err(XmlFunctionTagsFailure::EmptyFunctionName); - } - let function_body_start = &after_function_prefix[name_end + 1..]; - - let Some(function_body_end) = function_body_start.find(shape.function_close.as_str()) else { - return Err(XmlFunctionTagsFailure::UnclosedFunctionBlock { - function_name, - expected_close: shape.function_close.clone(), - }); - }; - let function_body = &function_body_start[..function_body_end]; - let after_function_close = - &function_body_start[function_body_end + shape.function_close.len()..]; - - let arguments_object = collect_parameters(function_body, shape, &function_name)?; - let arguments_value = serde_json::Value::Object(arguments_object); - let arguments = ToolCallArguments::from_string(arguments_value.to_string()); - - Ok(Some(( - ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - function_name, - arguments, - ), - after_function_close, - ))) -} - -/// # Errors -/// -/// Returns [`XmlFunctionTagsFailure`] when the body looks like an XML -/// function-tag tool-call block (matches the function open prefix) but -/// contains a structural issue: empty function/parameter name or an -/// unclosed function/parameter block. -pub fn parse( - body: &str, - shape: &XmlTagsShape, -) -> Result, XmlFunctionTagsFailure> { - if !shape_is_complete(shape) { - return Ok(Vec::new()); - } - - let mut parsed = Vec::new(); - let mut remaining = body; - - while let Some((call, rest)) = parse_one_function(remaining, shape)? { - parsed.push(call); - remaining = rest; - } - - Ok(parsed) -} - -#[cfg(test)] -mod tests { - use serde_json::json; - - use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; - use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; - use llama_cpp_bindings_types::xml_tags_shape::XmlTagsShape; - - use super::parse; - use crate::error::xml_function_tags_failure::XmlFunctionTagsFailure; - - fn xml_shape() -> XmlTagsShape { - XmlTagsShape { - function_open_prefix: "".to_owned(), - parameter_open_prefix: "".to_owned(), - } - } - - #[test] - fn parses_single_function_with_one_parameter() { - let body = - "\n\n\nParis\n\n\n"; - let parsed = parse(body, &xml_shape()).expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "get_weather"); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ); - } - - #[test] - fn parses_function_with_multiple_parameters() { - let body = "1two"; - let parsed = parse(body, &xml_shape()).expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"a": 1, "b": "two"})), - ); - } - - #[test] - fn parses_two_function_blocks_in_one_body() { - let body = "12"; - let parsed = parse(body, &xml_shape()).expect("must parse"); - - assert_eq!(parsed.len(), 2); - assert_eq!(parsed[0].name, "a"); - assert_eq!(parsed[1].name, "b"); - } - - #[test] - fn preserves_multi_line_parameter_value() { - let body = "\n\nline one\nline two\n\n"; - let parsed = parse(body, &xml_shape()).expect("must parse"); - - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"msg": "line one\nline two"})), - ); - } - - #[test] - fn rejects_function_tag_missing_closing_angle_with_typed_failure() { - let body = "Paris"; - - assert_eq!( - parse(body, &xml_shape()), - Err(XmlFunctionTagsFailure::UnclosedFunctionBlock { - function_name: String::new(), - expected_close: "".to_owned(), - }), - ); - } - - #[test] - fn rejects_function_block_missing_close_tag_with_typed_failure() { - let body = "Paris"; - - assert_eq!( - parse(body, &xml_shape()), - Err(XmlFunctionTagsFailure::UnclosedFunctionBlock { - function_name: "get_weather".to_owned(), - expected_close: "".to_owned(), - }), - ); - } - - #[test] - fn rejects_parameter_tag_missing_closing_angle_with_typed_failure() { - let body = ""; - - assert_eq!( - parse(body, &xml_shape()), - Err(XmlFunctionTagsFailure::UnclosedParameterBlock { - function_name: "f".to_owned(), - parameter_name: String::new(), - expected_close: "".to_owned(), - }), - ); - } - - #[test] - fn rejects_parameter_block_missing_close_tag_with_typed_failure() { - let body = "Paris"; - - assert_eq!( - parse(body, &xml_shape()), - Err(XmlFunctionTagsFailure::UnclosedParameterBlock { - function_name: "get_weather".to_owned(), - parameter_name: "location".to_owned(), - expected_close: "".to_owned(), - }), - ); - } - - #[test] - fn rejects_empty_function_name_with_typed_failure() { - let body = "1"; - - assert_eq!( - parse(body, &xml_shape()), - Err(XmlFunctionTagsFailure::EmptyFunctionName), - ); - } - - #[test] - fn rejects_empty_parameter_name_with_typed_failure() { - let body = "1"; - - assert_eq!( - parse(body, &xml_shape()), - Err(XmlFunctionTagsFailure::EmptyParameterName { - function_name: "f".to_owned(), - }), - ); - } - - #[test] - fn returns_empty_when_body_has_no_function_tag() { - let parsed = - parse("plain text without function tags", &xml_shape()).expect("must parse empty"); - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn returns_empty_when_shape_has_empty_required_field() { - let mut shape = xml_shape(); - shape.function_close.clear(); - let body = "1"; - let parsed = parse(body, &shape).expect("must parse empty"); - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn parses_newline_padded_parameters_inside_a_tool_call_block_into_typed_arguments() { - let body = "\n\ -\n\ -\n\ -tuna\n\ -\n\ -\n\ -8\n\ -\n\ -\n\ -get off the keyboard\n\ -\n\ -\n\ -"; - let parsed = parse(body, &xml_shape()).expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "negotiate_with_cat"); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({ - "bribe": "tuna", - "desperation_level": 8, - "topic": "get off the keyboard", - })), - ); - } -} From 680330f84a58bdd2e9eed0ea92ae69aaccfe7be8 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 07:30:55 +0100 Subject: [PATCH 43/82] Remove the NextN hidden-state API built on llama.cpp's staging header and track the embeddings flag in Rust --- llama-cpp-bindings-sys/README.md | 7 + llama-cpp-bindings-sys/wrapper_context.cpp | 73 --- llama-cpp-bindings-sys/wrapper_context.h | 26 -- .../tests/embeddings_toggling.rs | 146 ++++++ .../tests/hidden_state_extraction.rs | 414 ------------------ llama-cpp-bindings-tests/tests/main.rs | 2 +- llama-cpp-bindings/src/context.rs | 323 +------------- .../src/error/embeddings_error.rs | 31 -- 8 files changed, 162 insertions(+), 860 deletions(-) create mode 100644 llama-cpp-bindings-tests/tests/embeddings_toggling.rs delete mode 100644 llama-cpp-bindings-tests/tests/hidden_state_extraction.rs diff --git a/llama-cpp-bindings-sys/README.md b/llama-cpp-bindings-sys/README.md index 697aba12d..f602675fd 100644 --- a/llama-cpp-bindings-sys/README.md +++ b/llama-cpp-bindings-sys/README.md @@ -1,3 +1,10 @@ # llama-cpp-bindings-sys Low level FFI bindings to [llama.cpp](https://github.com/ggerganov/llama.cpp). + +## Private llama.cpp headers + +The wrappers use llama.cpp's public headers, with two deliberate exceptions where llama.cpp exposes no public equivalent: + +- `src/llama-model.h` (`wrapper_context.cpp`): reads the model's default causal attention before a context exists. Context creation aborts the process on a `GGML_ASSERT` when the context's sequences exceed its output capacity, and that capacity depends on whether attention is causal, so the bindings validate it first. `llama_get_causal_attn` only answers once a context exists. +- `src/llama-grammar.h` (`wrapper_gbnf.cpp`): validates GBNF without a model and reports why a grammar is rejected (syntax error, empty rule set, missing root symbol, unsupported grammar). `llama_sampler_init_grammar` only returns `NULL` and writes the reason to stderr. diff --git a/llama-cpp-bindings-sys/wrapper_context.cpp b/llama-cpp-bindings-sys/wrapper_context.cpp index a07d50df4..7b8d7d363 100644 --- a/llama-cpp-bindings-sys/wrapper_context.cpp +++ b/llama-cpp-bindings-sys/wrapper_context.cpp @@ -5,80 +5,7 @@ #include #include -#include -#include -#include #include -#include "wrapper_report_current_exception.h" - -extern "C" auto llama_rs_context_embeddings_enabled(const struct llama_context * ctx) -> bool { - return ctx->get_cparams().embeddings; -} - -extern "C" auto llama_rs_context_embeddings_nextn_enabled(const struct llama_context * ctx) -> bool { - return ctx->get_cparams().embeddings_nextn; -} - -extern "C" auto llama_rs_context_embeddings_nextn_masked(const struct llama_context * ctx) -> bool { - return ctx->get_cparams().embeddings_nextn_masked; -} - -extern "C" void llama_rs_set_embeddings_nextn(struct llama_context * ctx, bool value, bool masked) { - llama_set_embeddings_nextn(ctx, value, masked); -} - -extern "C" auto llama_rs_get_embeddings_nextn_ith(struct llama_context * ctx, int32_t token_index) -> float * { - return llama_get_embeddings_nextn_ith(ctx, token_index); -} - -extern "C" auto llama_rs_context_graph_produces_nextn_hidden_state( - struct llama_context * ctx, - bool * out_produces_nextn_hidden_state, - char ** out_error) -> llama_rs_context_nextn_hidden_state_probe_status { - if (out_produces_nextn_hidden_state != nullptr) { - *out_produces_nextn_hidden_state = false; - } - if (out_error != nullptr) { - *out_error = nullptr; - } - if (ctx == nullptr) { - return LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_CTX_ARG; - } - if (out_produces_nextn_hidden_state == nullptr) { - return LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_PRODUCES_NEXTN_HIDDEN_STATE_ARG; - } - if (out_error == nullptr) { - return LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_ERROR_ARG; - } - try { - constexpr uint32_t smallest_ubatch_tokens = 1; - constexpr uint32_t smallest_ubatch_sequences = 1; - constexpr uint32_t smallest_ubatch_outputs = 1; - constexpr bool split_graph_without_allocating_buffers = true; - - llama_memory_context_ptr memory_context; - llama_memory_t memory = llama_get_memory(ctx); - if (memory != nullptr) { - memory_context = memory->init_full(); - } - if (ctx->graph_reserve( - smallest_ubatch_tokens, - smallest_ubatch_sequences, - smallest_ubatch_outputs, - memory_context.get(), - split_graph_without_allocating_buffers) == nullptr) { - return LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_FAILED_TO_RESERVE_GRAPH; - } - *out_produces_nextn_hidden_state = ctx->get_gf_res_reserve()->get_h_nextn() != nullptr; - return LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_ERROR_STRING_ALLOCATION_FAILED); - } -} extern "C" auto llama_rs_model_causal_attn(const struct llama_model * model) -> bool { return model->hparams.causal_attn; diff --git a/llama-cpp-bindings-sys/wrapper_context.h b/llama-cpp-bindings-sys/wrapper_context.h index 90db33852..7516e6bbf 100644 --- a/llama-cpp-bindings-sys/wrapper_context.h +++ b/llama-cpp-bindings-sys/wrapper_context.h @@ -12,32 +12,6 @@ struct llama_adapter_lora; struct llama_context; struct llama_model; -typedef enum llama_rs_context_nextn_hidden_state_probe_status { - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_OK = 0, - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_CTX_ARG, - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_PRODUCES_NEXTN_HIDDEN_STATE_ARG, - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_ERROR_ARG, - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_FAILED_TO_RESERVE_GRAPH, - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_context_nextn_hidden_state_probe_status; - -bool llama_rs_context_embeddings_enabled(const struct llama_context * ctx); - -bool llama_rs_context_embeddings_nextn_enabled(const struct llama_context * ctx); - -bool llama_rs_context_embeddings_nextn_masked(const struct llama_context * ctx); - -void llama_rs_set_embeddings_nextn(struct llama_context * ctx, bool value, bool masked); - -float * llama_rs_get_embeddings_nextn_ith(struct llama_context * ctx, int32_t token_index); - -llama_rs_context_nextn_hidden_state_probe_status llama_rs_context_graph_produces_nextn_hidden_state( - struct llama_context * ctx, - bool * out_produces_nextn_hidden_state, - char ** out_error); - bool llama_rs_model_causal_attn(const struct llama_model * model); bool llama_rs_context_produced_logits(struct llama_context * ctx); diff --git a/llama-cpp-bindings-tests/tests/embeddings_toggling.rs b/llama-cpp-bindings-tests/tests/embeddings_toggling.rs new file mode 100644 index 000000000..32807fa50 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/embeddings_toggling.rs @@ -0,0 +1,146 @@ +use anyhow::Result; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +const STATE: &str = "Shoes arrived two weeks late and in the wrong size."; +const ROW: &str = " Which team should handle this ticket?"; +const OTHER_STATE: &str = "The weather in Oslo is cold, dark and windy all week."; + +fn tokenize(fixture: &LlamaFixture<'_>, text: &str) -> Result> { + Ok(fixture + .model + .str_to_token(text, AddBos::Never, ParseSpecialTokens::Never)?) +} + +fn add_tokens( + batch: &mut LlamaBatch, + tokens: &[LlamaToken], + first_position: i32, + sequence_id: i32, + output_on_last_token: bool, +) -> Result<()> { + let last_index = tokens.len() - 1; + + for (index, token) in tokens.iter().enumerate() { + batch.add( + &SampledToken::Content(*token), + first_position + i32::try_from(index)?, + &[sequence_id], + output_on_last_token && index == last_index, + )?; + } + + Ok(()) +} + +fn decode_tokens( + context: &mut LlamaContext<'_>, + tokens: &[LlamaToken], + first_position: i32, + output_on_last_token: bool, +) -> Result { + let mut batch = LlamaBatch::new(ContextLimits::of(context)); + + add_tokens(&mut batch, tokens, first_position, 0, output_on_last_token)?; + context.decode(&batch)?; + + Ok(batch.n_tokens() - 1) +} + +fn final_hidden_state_context<'fixture>( + fixture: &'fixture LlamaFixture<'_>, + embeddings: bool, +) -> Result> { + Ok(LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params) + .into_llama_context_params() + .with_embeddings(embeddings) + .with_pooling_type(LlamaPoolingType::None), + )?) +} + +fn row_hidden_state_after_enabling_embeddings( + fixture: &LlamaFixture<'_>, + state: &[LlamaToken], + row: &[LlamaToken], +) -> Result> { + let mut context = final_hidden_state_context(fixture, false)?; + + decode_tokens(&mut context, state, 0, false)?; + context.set_embeddings(true); + + let row_last_index = decode_tokens(&mut context, row, i32::try_from(state.len())?, true)?; + + Ok(context.embeddings_ith(row_last_index)?.to_vec()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, +)] +fn final_hidden_states_after_enabling_embeddings_for_a_row_equal_those_of_an_embeddings_context( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let state = tokenize(fixture, STATE)?; + let row = tokenize(fixture, ROW)?; + let toggled_row_hidden_state = + row_hidden_state_after_enabling_embeddings(fixture, &state, &row)?; + + let mut embeddings_context = final_hidden_state_context(fixture, true)?; + + decode_tokens(&mut embeddings_context, &state, 0, false)?; + + let row_last_index = decode_tokens( + &mut embeddings_context, + &row, + i32::try_from(state.len())?, + true, + )?; + + assert_eq!( + toggled_row_hidden_state.as_slice(), + embeddings_context.embeddings_ith(row_last_index)?, + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, +)] +fn final_hidden_states_of_a_row_follow_the_state_decoded_before_it( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let row = tokenize(fixture, ROW)?; + + assert_ne!( + row_hidden_state_after_enabling_embeddings(fixture, &tokenize(fixture, STATE)?, &row)?, + row_hidden_state_after_enabling_embeddings( + fixture, + &tokenize(fixture, OTHER_STATE)?, + &row + )?, + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs b/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs deleted file mode 100644 index 3f8f531ff..000000000 --- a/llama-cpp-bindings-tests/tests/hidden_state_extraction.rs +++ /dev/null @@ -1,414 +0,0 @@ -use anyhow::Result; - -use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_bindings::context::context_limits::ContextLimits; -use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; -use llama_cpp_bindings::error::embeddings_error::EmbeddingsError; -use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::add_bos::AddBos; -use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; -use llama_cpp_bindings::sampled_token::SampledToken; -use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_test_harness::llama_fixture::LlamaFixture; -use llama_cpp_test_harness_macros::llama_test; - -const STATE: &str = "Shoes arrived two weeks late and in the wrong size."; -const FIRST_BRANCH: &str = " Which team should handle this ticket?"; -const SECOND_BRANCH: &str = " Is the customer angry about the delay?"; -const OTHER_STATE: &str = "The weather in Oslo is cold, dark and windy all week."; - -fn tokenize(fixture: &LlamaFixture<'_>, text: &str) -> Result> { - Ok(fixture - .model - .str_to_token(text, AddBos::Never, ParseSpecialTokens::Never)?) -} - -fn hidden_state_context<'fixture>( - fixture: &'fixture LlamaFixture<'_>, -) -> Result> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_pooling_type(LlamaPoolingType::None) - .with_kv_unified(true), - )?; - - context.enable_masked_nextn_embeddings()?; - - Ok(context) -} - -fn add_tokens( - batch: &mut LlamaBatch, - tokens: &[LlamaToken], - first_position: i32, - sequence_id: i32, - output_on_last_token: bool, -) -> Result<()> { - let last_index = tokens.len() - 1; - - for (index, token) in tokens.iter().enumerate() { - batch.add( - &SampledToken::Content(*token), - first_position + i32::try_from(index)?, - &[sequence_id], - output_on_last_token && index == last_index, - )?; - } - - Ok(()) -} - -fn hidden_state_after( - fixture: &LlamaFixture<'_>, - state: &[LlamaToken], - branch: &[LlamaToken], -) -> Result> { - let mut context = hidden_state_context(fixture)?; - let mut state_batch = LlamaBatch::new(ContextLimits::of(&context)); - - add_tokens(&mut state_batch, state, 0, 0, false)?; - context.decode(&state_batch)?; - - let mut branch_batch = LlamaBatch::new(ContextLimits::of(&context)); - - add_tokens( - &mut branch_batch, - branch, - i32::try_from(state.len())?, - 0, - true, - )?; - context.decode(&branch_batch)?; - - Ok(context - .nextn_embeddings_ith(branch_batch.n_tokens() - 1)? - .to_vec()) -} - -fn context_with_a_decoded_fork<'fixture>( - fixture: &'fixture LlamaFixture<'_>, - state: &[LlamaToken], - branch: &[LlamaToken], -) -> Result> { - let mut context = hidden_state_context(fixture)?; - let mut state_batch = LlamaBatch::new(ContextLimits::of(&context)); - - add_tokens(&mut state_batch, state, 0, 0, false)?; - context.decode(&state_batch)?; - context.memory_seq_cp(0, 1, None, None)?; - - let mut branch_batch = LlamaBatch::new(ContextLimits::of(&context)); - - add_tokens( - &mut branch_batch, - branch, - i32::try_from(state.len())?, - 1, - true, - )?; - context.decode(&branch_batch)?; - - Ok(context) -} - -fn decode_tokens( - context: &mut LlamaContext<'_>, - tokens: &[LlamaToken], - first_position: i32, - output_on_last_token: bool, -) -> Result { - let mut batch = LlamaBatch::new(ContextLimits::of(context)); - - add_tokens(&mut batch, tokens, first_position, 0, output_on_last_token)?; - context.decode(&batch)?; - - Ok(batch.n_tokens() - 1) -} - -fn final_hidden_state_context<'fixture>( - fixture: &'fixture LlamaFixture<'_>, - embeddings: bool, -) -> Result> { - Ok(LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_embeddings(embeddings) - .with_pooling_type(LlamaPoolingType::None), - )?) -} - -fn row_hidden_state_after_enabling_embeddings( - fixture: &LlamaFixture<'_>, - state: &[LlamaToken], - row: &[LlamaToken], -) -> Result> { - let mut context = final_hidden_state_context(fixture, false)?; - - decode_tokens(&mut context, state, 0, false)?; - context.set_embeddings(true); - - let row_last_index = decode_tokens(&mut context, row, i32::try_from(state.len())?, true)?; - - Ok(context.embeddings_ith(row_last_index)?.to_vec()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, -)] -fn masked_nextn_embeddings_equal_the_final_hidden_states(fixture: &LlamaFixture<'_>) -> Result<()> { - let tokens = tokenize(fixture, STATE)?; - let last_index = i32::try_from(tokens.len() - 1)?; - - let mut embedding_context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_embeddings(true) - .with_pooling_type(LlamaPoolingType::None), - )?; - let mut embedding_batch = LlamaBatch::new(ContextLimits::of(&embedding_context)); - - embedding_batch.add_sequence(&tokens, 0, true)?; - embedding_context.decode(&embedding_batch)?; - - let mut nextn_context = hidden_state_context(fixture)?; - let mut nextn_batch = LlamaBatch::new(ContextLimits::of(&nextn_context)); - - nextn_batch.add_sequence(&tokens, 0, true)?; - nextn_context.decode(&nextn_batch)?; - - assert_eq!( - nextn_context.nextn_embeddings_ith(last_index)?, - embedding_context.embeddings_ith(last_index)?, - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, -)] -fn masked_nextn_embeddings_exist_only_for_output_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { - let tokens = tokenize(fixture, STATE)?; - let mut context = hidden_state_context(fixture)?; - let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - - add_tokens(&mut batch, &tokens, 0, 0, true)?; - context.decode(&batch)?; - - assert_eq!( - context.nextn_embeddings_ith(0), - Err(EmbeddingsError::NextnEmbeddingUnavailable { token_index: 0 }) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, -)] -fn nextn_embeddings_refuse_a_pooling_context(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_pooling_type(LlamaPoolingType::Mean), - )?; - - assert_eq!( - context.enable_masked_nextn_embeddings(), - Err(EmbeddingsError::NextnEmbeddingsRequireNonePooling { - pooling_type: LlamaPoolingType::Mean, - }) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, -)] -fn nextn_embeddings_are_unavailable_until_enabled(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = fixture.build_context()?; - - assert_eq!( - context.nextn_embeddings_ith(0), - Err(EmbeddingsError::NextnEmbeddingsNotEnabled) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, - n_seq_max = 2, -)] -fn a_forked_branch_has_the_hidden_state_of_an_independent_decode( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let state = tokenize(fixture, STATE)?; - let first_branch = tokenize(fixture, FIRST_BRANCH)?; - let context = context_with_a_decoded_fork(fixture, &state, &first_branch)?; - - assert_eq!( - context.nextn_embeddings_ith(i32::try_from(first_branch.len())? - 1)?, - hidden_state_after(fixture, &state, &first_branch)?.as_slice(), - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, - n_seq_max = 2, -)] -fn a_released_fork_leaves_its_shared_state_as_an_independent_decode_would( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let state = tokenize(fixture, STATE)?; - let second_branch = tokenize(fixture, SECOND_BRANCH)?; - let mut context = - context_with_a_decoded_fork(fixture, &state, &tokenize(fixture, FIRST_BRANCH)?)?; - - context.memory_seq_rm(Some(1), None, None)?; - - let mut second_branch_batch = LlamaBatch::new(ContextLimits::of(&context)); - - add_tokens( - &mut second_branch_batch, - &second_branch, - i32::try_from(state.len())?, - 0, - true, - )?; - context.decode(&second_branch_batch)?; - - assert_eq!( - context.nextn_embeddings_ith(second_branch_batch.n_tokens() - 1)?, - hidden_state_after(fixture, &state, &second_branch)?.as_slice(), - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, -)] -fn final_hidden_states_after_enabling_embeddings_for_a_row_equal_those_of_an_embeddings_context( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let state = tokenize(fixture, STATE)?; - let row = tokenize(fixture, FIRST_BRANCH)?; - let toggled_row_hidden_state = - row_hidden_state_after_enabling_embeddings(fixture, &state, &row)?; - - let mut embeddings_context = final_hidden_state_context(fixture, true)?; - - decode_tokens(&mut embeddings_context, &state, 0, false)?; - - let row_last_index = decode_tokens( - &mut embeddings_context, - &row, - i32::try_from(state.len())?, - true, - )?; - - assert_eq!( - toggled_row_hidden_state.as_slice(), - embeddings_context.embeddings_ith(row_last_index)?, - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, -)] -fn final_hidden_states_of_a_row_follow_the_state_decoded_before_it( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let row = tokenize(fixture, FIRST_BRANCH)?; - - assert_ne!( - row_hidden_state_after_enabling_embeddings(fixture, &tokenize(fixture, STATE)?, &row)?, - row_hidden_state_after_enabling_embeddings( - fixture, - &tokenize(fixture, OTHER_STATE)?, - &row - )?, - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, -)] -fn nextn_embeddings_are_refused_and_stay_disabled_on_a_model_without_nextn_hidden_states( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = final_hidden_state_context(fixture, false)?; - - assert_eq!( - context.enable_masked_nextn_embeddings(), - Err(EmbeddingsError::NextnEmbeddingsUnsupported) - ); - assert_eq!( - context.nextn_embeddings_ith(0), - Err(EmbeddingsError::NextnEmbeddingsNotEnabled) - ); - - Ok(()) -} diff --git a/llama-cpp-bindings-tests/tests/main.rs b/llama-cpp-bindings-tests/tests/main.rs index b3c82f992..ceaa6a1cd 100644 --- a/llama-cpp-bindings-tests/tests/main.rs +++ b/llama-cpp-bindings-tests/tests/main.rs @@ -18,12 +18,12 @@ mod embedding_availability; mod embedding_generation; mod embedding_retrieval_errors; mod embedding_row_width; +mod embeddings_toggling; mod encode_batch_rejection; mod encoder_requirement; mod end_of_generation_token_detection; mod generation_end_classification; mod grammar_sampler; -mod hidden_state_extraction; mod image_question_answering; mod json_schema_constrained_generation; mod lazy_grammar_sampler; diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index b46bd91a4..9c96e6ee9 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -65,63 +65,6 @@ const fn check_lora_adapters_result(err_code: i32) -> Result<(), LlamaLoraAdapte Ok(()) } -fn nextn_hidden_state_probe_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_context_nextn_hidden_state_probe_status, - produces_nextn_hidden_state: bool, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_OK => { - Ok(produces_nextn_hidden_state) - } - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_CTX_ARG => { - Err(FfiContractError { - operation: "llama_rs_context_graph_produces_nextn_hidden_state", - detail: "was given a null ctx argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_PRODUCES_NEXTN_HIDDEN_STATE_ARG => { - Err(FfiContractError { - operation: "llama_rs_context_graph_produces_nextn_hidden_state", - detail: "was given a null out_produces_nextn_hidden_state argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_ERROR_ARG => { - Err(FfiContractError { - operation: "llama_rs_context_graph_produces_nextn_hidden_state", - detail: "was given a null out_error argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_FAILED_TO_RESERVE_GRAPH => { - Err(EmbeddingsError::NextnProbeGraphReservationFailed) - } - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_ERROR_STRING_ALLOCATION_FAILED => { - Err(EmbeddingsError::NextnProbeNotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(EmbeddingsError::NextnProbeLlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - out_error, - "llama_rs_context_graph_produces_nextn_hidden_state", - "reported a thrown C++ exception without an error message", - ) - }?; - Err(EmbeddingsError::NextnProbeReported { message }) - } - other => Err(FfiStatusError { - operation: "llama_rs_context_graph_produces_nextn_hidden_state", - code: i64::from(other), - } - .into()), - } -} - fn new_context_with_model_status_to_result( status: llama_cpp_bindings_sys::llama_rs_new_context_with_model_status, out_ctx: *mut llama_cpp_bindings_sys::llama_context, @@ -337,6 +280,7 @@ pub struct LlamaContext<'model> { batch_ext: NonNull, limits: ContextLimits, keeps_no_memory: bool, + embeddings: bool, } impl Debug for LlamaContext<'_> { @@ -351,6 +295,7 @@ impl<'model> LlamaContext<'model> { fn with_batch( model: &'model LlamaModel, context: NonNull, + embeddings: bool, ) -> Result { let mut out_batch: *mut llama_cpp_bindings_sys::llama_batch_ext = ptr::null_mut(); let mut out_error: *mut c_char = ptr::null_mut(); @@ -387,6 +332,7 @@ impl<'model> LlamaContext<'model> { n_vocab: model.n_vocab(), }, keeps_no_memory, + embeddings, }) } @@ -400,6 +346,7 @@ impl<'model> LlamaContext<'model> { params: LlamaContextParams, ) -> Result { let context_params = params.context_params; + let embeddings = params.embeddings(); SequenceOutputCapacity { attention_type: params.attention_type(), @@ -430,7 +377,7 @@ impl<'model> LlamaContext<'model> { }; let context = new_context_with_model_status_to_result(status, out_ctx, out_error)?; - Self::with_batch(model, context) + Self::with_batch(model, context, embeddings) } #[must_use] @@ -589,23 +536,11 @@ impl<'model> LlamaContext<'model> { unsafe { llama_cpp_bindings_sys::llama_set_embeddings(self.context.as_ptr(), embeddings); } + self.embeddings = embeddings; } - fn embeddings_enabled(&self) -> bool { - unsafe { - llama_cpp_bindings_sys::llama_rs_context_embeddings_enabled(self.context.as_ptr()) - } - } - - fn masked_nextn_embeddings_enabled(&self) -> bool { - let nextn_enabled = unsafe { - llama_cpp_bindings_sys::llama_rs_context_embeddings_nextn_enabled(self.context.as_ptr()) - }; - let nextn_masked = unsafe { - llama_cpp_bindings_sys::llama_rs_context_embeddings_nextn_masked(self.context.as_ptr()) - }; - - nextn_enabled && nextn_masked + const fn embeddings_enabled(&self) -> bool { + self.embeddings } fn output_embedding_width(&self) -> Result { @@ -683,83 +618,6 @@ impl<'model> LlamaContext<'model> { } } - /// # Errors - /// - /// - When the context pools its outputs, because llama.cpp only extracts `NextN` embeddings - /// with `LLAMA_POOLING_TYPE_NONE`. - /// - When the model's graph produces no `NextN` hidden state; `NextN` embeddings stay disabled. - /// - When llama.cpp fails while checking the graph; `NextN` embeddings stay disabled. - pub fn enable_masked_nextn_embeddings(&mut self) -> Result<(), EmbeddingsError> { - let pooling_type = self.pooling_type(); - - if pooling_type != LlamaPoolingType::None { - return Err(EmbeddingsError::NextnEmbeddingsRequireNonePooling { pooling_type }); - } - - self.set_masked_nextn_embeddings(true); - - let enabled = - self.graph_produces_nextn_hidden_state() - .and_then(|produces_nextn_hidden_state| { - produces_nextn_hidden_state - .then_some(()) - .ok_or(EmbeddingsError::NextnEmbeddingsUnsupported) - }); - - if enabled.is_err() { - self.set_masked_nextn_embeddings(false); - } - - enabled - } - - fn set_masked_nextn_embeddings(&mut self, enabled: bool) { - unsafe { - llama_cpp_bindings_sys::llama_rs_set_embeddings_nextn( - self.context.as_ptr(), - enabled, - enabled, - ); - } - } - - fn graph_produces_nextn_hidden_state(&self) -> Result { - let mut produces_nextn_hidden_state = false; - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_context_graph_produces_nextn_hidden_state( - self.context.as_ptr(), - &raw mut produces_nextn_hidden_state, - &raw mut out_error, - ) - }; - - nextn_hidden_state_probe_status_to_result(status, produces_nextn_hidden_state, out_error) - } - - /// # Errors - /// - /// - When masked `NextN` embeddings were not enabled on this context. - /// - When the given token was not marked as an output of the last decoded batch. - pub fn nextn_embeddings_ith(&self, token_index: i32) -> Result<&[f32], EmbeddingsError> { - if !self.masked_nextn_embeddings_enabled() { - return Err(EmbeddingsError::NextnEmbeddingsNotEnabled); - } - - let embedding = unsafe { - llama_cpp_bindings_sys::llama_rs_get_embeddings_nextn_ith( - self.context.as_ptr(), - token_index, - ) - }; - - if embedding.is_null() { - Err(EmbeddingsError::NextnEmbeddingUnavailable { token_index }) - } else { - self.output_embedding_row(embedding) - } - } - fn ensure_logits_produced(&self) -> Result<(), LogitsError> { if unsafe { llama_cpp_bindings_sys::llama_rs_context_produced_logits(self.context.as_ptr()) @@ -891,171 +749,6 @@ mod check_lora_adapters_result_tests { } } -#[cfg(test)] -mod nextn_hidden_state_probe_status_to_result_tests { - use std::ptr; - - use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; - use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; - use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; - use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer::wrapper_allocated_error_message_pointer; - - use super::nextn_hidden_state_probe_status_to_result; - use crate::error::embeddings_error::EmbeddingsError; - - #[test] - fn nextn_hidden_state_probe_status_to_result_maps_ok_to_whether_the_graph_produces_it() { - assert_eq!( - [true, false].map(|produces_nextn_hidden_state| { - nextn_hidden_state_probe_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_OK, - produces_nextn_hidden_state, - ptr::null_mut(), - ) - }), - [Ok(true), Ok(false)] - ); - } - - #[test] - fn nextn_hidden_state_probe_status_to_result_reports_a_null_ctx_arg_as_a_contract_violation() { - assert_eq!( - nextn_hidden_state_probe_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_CTX_ARG, - false, - ptr::null_mut(), - ), - Err(FfiContractError { - operation: "llama_rs_context_graph_produces_nextn_hidden_state", - detail: "was given a null ctx argument", - } - .into()) - ); - } - - #[test] - fn nextn_hidden_state_probe_status_to_result_reports_a_null_out_produces_nextn_hidden_state_arg_as_a_contract_violation() - { - assert_eq!( - nextn_hidden_state_probe_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_PRODUCES_NEXTN_HIDDEN_STATE_ARG, - false, - ptr::null_mut(), - ), - Err(FfiContractError { - operation: "llama_rs_context_graph_produces_nextn_hidden_state", - detail: "was given a null out_produces_nextn_hidden_state argument", - } - .into()) - ); - } - - #[test] - fn nextn_hidden_state_probe_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() - { - assert_eq!( - nextn_hidden_state_probe_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_ERROR_ARG, - false, - ptr::null_mut(), - ), - Err(FfiContractError { - operation: "llama_rs_context_graph_produces_nextn_hidden_state", - detail: "was given a null out_error argument", - } - .into()) - ); - } - - #[test] - fn nextn_hidden_state_probe_status_to_result_reports_a_failed_graph_reservation() { - assert_eq!( - nextn_hidden_state_probe_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_FAILED_TO_RESERVE_GRAPH, - false, - ptr::null_mut(), - ), - Err(EmbeddingsError::NextnProbeGraphReservationFailed) - ); - } - - #[test] - fn nextn_hidden_state_probe_status_to_result_reports_not_enough_memory_for_the_error_string() { - assert_eq!( - nextn_hidden_state_probe_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_ERROR_STRING_ALLOCATION_FAILED, - false, - ptr::null_mut(), - ), - Err(EmbeddingsError::NextnProbeNotEnoughMemory) - ); - } - - #[test] - fn nextn_hidden_state_probe_status_to_result_reports_llama_cpp_running_out_of_memory() { - assert_eq!( - nextn_hidden_state_probe_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_OUT_OF_MEMORY, - false, - ptr::null_mut(), - ), - Err(EmbeddingsError::NextnProbeLlamaCppOutOfMemory) - ); - } - - #[test] - fn nextn_hidden_state_probe_status_to_result_maps_a_thrown_exception_with_its_message() { - let expected_message = unsafe { - read_and_free_cpp_string( - wrapper_allocated_error_message_pointer(), - "llama_rs_context_graph_produces_nextn_hidden_state", - "reported a thrown C++ exception without an error message", - ) - }; - - assert_eq!( - nextn_hidden_state_probe_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_THREW_CXX_EXCEPTION, - false, - wrapper_allocated_error_message_pointer(), - ) - .err(), - expected_message - .ok() - .map(|message| EmbeddingsError::NextnProbeReported { message }) - ); - } - - #[test] - fn nextn_hidden_state_probe_status_to_result_reports_an_exception_without_a_message_as_a_contract_violation() - { - assert_eq!( - nextn_hidden_state_probe_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_THREW_CXX_EXCEPTION, - false, - ptr::null_mut(), - ), - Err(FfiContractError { - operation: "llama_rs_context_graph_produces_nextn_hidden_state", - detail: "reported a thrown C++ exception without an error message", - } - .into()) - ); - } - - #[test] - fn nextn_hidden_state_probe_status_to_result_preserves_an_unknown_status() { - assert_eq!( - nextn_hidden_state_probe_status_to_result(255, false, ptr::null_mut()), - Err(FfiStatusError { - operation: "llama_rs_context_graph_produces_nextn_hidden_state", - code: 255, - } - .into()) - ); - } -} - #[cfg(test)] mod new_context_with_model_status_to_result_tests { use std::ptr; diff --git a/llama-cpp-bindings/src/error/embeddings_error.rs b/llama-cpp-bindings/src/error/embeddings_error.rs index f9d2f3ed2..313c6ced2 100644 --- a/llama-cpp-bindings/src/error/embeddings_error.rs +++ b/llama-cpp-bindings/src/error/embeddings_error.rs @@ -1,6 +1,3 @@ -use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; -use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; - #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum EmbeddingsError { #[error("Embeddings weren't enabled in the context options")] @@ -23,32 +20,4 @@ pub enum EmbeddingsError { }, #[error("Invalid embedding dimension: {0}")] InvalidEmbeddingDimension(#[source] std::num::TryFromIntError), - #[error( - "NextN embeddings need a context with LLAMA_POOLING_TYPE_NONE, but it pools with {pooling_type:?}" - )] - NextnEmbeddingsRequireNonePooling { - pooling_type: crate::context::llama_pooling_type::LlamaPoolingType, - }, - #[error("NextN embeddings weren't enabled on this context")] - NextnEmbeddingsNotEnabled, - #[error( - "NextN embeddings are unavailable because this model's graph produces no NextN hidden state" - )] - NextnEmbeddingsUnsupported, - #[error("llama.cpp failed to reserve the graph that checks for a NextN hidden state")] - NextnProbeGraphReservationFailed, - #[error("not enough memory to report why the NextN hidden state check failed")] - NextnProbeNotEnoughMemory, - #[error("llama.cpp ran out of memory while checking for a NextN hidden state")] - NextnProbeLlamaCppOutOfMemory, - #[error("llama.cpp failed while checking for a NextN hidden state: {message}")] - NextnProbeReported { message: String }, - #[error(transparent)] - FfiContract(#[from] FfiContractError), - #[error(transparent)] - FfiStatus(#[from] FfiStatusError), - #[error( - "No NextN embedding exists for token {token_index}; it was not marked as an output of the last decoded batch" - )] - NextnEmbeddingUnavailable { token_index: i32 }, } From ff4dfdf976bcca150cfcf2fc57f29d499736f4ca Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 07:31:15 +0100 Subject: [PATCH 44/82] Drop the private header notes from the sys README --- llama-cpp-bindings-sys/README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/llama-cpp-bindings-sys/README.md b/llama-cpp-bindings-sys/README.md index f602675fd..697aba12d 100644 --- a/llama-cpp-bindings-sys/README.md +++ b/llama-cpp-bindings-sys/README.md @@ -1,10 +1,3 @@ # llama-cpp-bindings-sys Low level FFI bindings to [llama.cpp](https://github.com/ggerganov/llama.cpp). - -## Private llama.cpp headers - -The wrappers use llama.cpp's public headers, with two deliberate exceptions where llama.cpp exposes no public equivalent: - -- `src/llama-model.h` (`wrapper_context.cpp`): reads the model's default causal attention before a context exists. Context creation aborts the process on a `GGML_ASSERT` when the context's sequences exceed its output capacity, and that capacity depends on whether attention is causal, so the bindings validate it first. `llama_get_causal_attn` only answers once a context exists. -- `src/llama-grammar.h` (`wrapper_gbnf.cpp`): validates GBNF without a model and reports why a grammar is rejected (syntax error, empty rule set, missing root symbol, unsupported grammar). `llama_sampler_init_grammar` only returns `NULL` and writes the reason to stderr. From 3432a4e4221692eb344d40795d849766f8de82e2 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 07:32:14 +0100 Subject: [PATCH 45/82] Report a token embedding without an output row as such, with its index --- .../tests/embedding_retrieval_errors.rs | 4 ++-- llama-cpp-bindings/src/context.rs | 6 +++--- llama-cpp-bindings/src/error/embeddings_error.rs | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs b/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs index a2626e4b0..575a55998 100644 --- a/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs +++ b/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs @@ -102,7 +102,7 @@ fn sequence_embedding_of_a_sequence_missing_from_the_decoded_batch_is_unavailabl n_ubatch = 512, embeddings = true, )] -fn token_embedding_of_an_index_without_a_decoded_output_reports_logits_not_enabled( +fn token_embedding_of_an_index_without_a_decoded_output_names_no_output_row( fixture: &LlamaFixture<'_>, ) -> Result<()> { let context = LlamaContext::from_model( @@ -115,7 +115,7 @@ fn token_embedding_of_an_index_without_a_decoded_output_reports_logits_not_enabl assert_eq!( context.embeddings_ith(999), - Err(EmbeddingsError::LogitsNotEnabled) + Err(EmbeddingsError::TokenNotAnOutput { token_index: 999 }) ); Ok(()) diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index 9c96e6ee9..7cdf0f0d1 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -594,8 +594,8 @@ impl<'model> LlamaContext<'model> { /// - When embeddings are not enabled on this context. /// - When the context pools its outputs, because llama.cpp then writes only sequence /// embeddings and never the token rows. - /// - When the given token didn't have logits enabled when it was passed. - /// - If the given token index exceeds the max token id. + /// - When `token_index` names no output row of the last decode. With embeddings enabled, + /// llama.cpp outputs every token of the batch. pub fn embeddings_ith(&self, token_index: i32) -> Result<&[f32], EmbeddingsError> { if !self.embeddings_enabled() { return Err(EmbeddingsError::NotEnabled); @@ -612,7 +612,7 @@ impl<'model> LlamaContext<'model> { }; if embedding.is_null() { - Err(EmbeddingsError::LogitsNotEnabled) + Err(EmbeddingsError::TokenNotAnOutput { token_index }) } else { self.output_embedding_row(embedding) } diff --git a/llama-cpp-bindings/src/error/embeddings_error.rs b/llama-cpp-bindings/src/error/embeddings_error.rs index 313c6ced2..e57a070ce 100644 --- a/llama-cpp-bindings/src/error/embeddings_error.rs +++ b/llama-cpp-bindings/src/error/embeddings_error.rs @@ -2,8 +2,8 @@ pub enum EmbeddingsError { #[error("Embeddings weren't enabled in the context options")] NotEnabled, - #[error("Logits were not enabled for the given token")] - LogitsNotEnabled, + #[error("token {token_index} names no output row of the last decode")] + TokenNotAnOutput { token_index: i32 }, #[error( "Sequence embeddings need a pooling context, but this context uses LLAMA_POOLING_TYPE_NONE" )] From fc2b60b6b19ba8a385a1eed3fa139630f185f3d1 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 07:33:53 +0100 Subject: [PATCH 46/82] Check in Rust that a model places one position per embedding before shifting sequence positions --- llama-cpp-bindings-sys/wrapper_common.cpp | 16 ---- llama-cpp-bindings-sys/wrapper_common.h | 4 - llama-cpp-bindings/src/context/memory.rs | 91 +++++------------------ llama-cpp-bindings/src/model/rope_type.rs | 25 +++++++ 4 files changed, 42 insertions(+), 94 deletions(-) diff --git a/llama-cpp-bindings-sys/wrapper_common.cpp b/llama-cpp-bindings-sys/wrapper_common.cpp index a32abfe94..75c93f116 100644 --- a/llama-cpp-bindings-sys/wrapper_common.cpp +++ b/llama-cpp-bindings-sys/wrapper_common.cpp @@ -217,14 +217,6 @@ extern "C" auto llama_rs_memory_seq_add( return LLAMA_RS_MEMORY_SEQ_ADD_NULL_CTX_ARG; } try { - const auto * model = llama_get_model(ctx); - if (model == nullptr) { - return LLAMA_RS_MEMORY_SEQ_ADD_NULL_MODEL; - } - const auto rope = llama_model_rope_type(model); - if (rope == LLAMA_ROPE_TYPE_MROPE || rope == LLAMA_ROPE_TYPE_VISION || rope == LLAMA_ROPE_TYPE_IMROPE) { - return LLAMA_RS_MEMORY_SEQ_ADD_INCOMPATIBLE_ROPE_TYPE; - } auto * mem = llama_get_memory(ctx); if (mem == nullptr) { return LLAMA_RS_MEMORY_SEQ_ADD_NULL_MEM; @@ -254,14 +246,6 @@ extern "C" auto llama_rs_memory_seq_div( return LLAMA_RS_MEMORY_SEQ_DIV_NULL_CTX_ARG; } try { - const auto * model = llama_get_model(ctx); - if (model == nullptr) { - return LLAMA_RS_MEMORY_SEQ_DIV_NULL_MODEL; - } - const auto rope = llama_model_rope_type(model); - if (rope == LLAMA_ROPE_TYPE_MROPE || rope == LLAMA_ROPE_TYPE_VISION || rope == LLAMA_ROPE_TYPE_IMROPE) { - return LLAMA_RS_MEMORY_SEQ_DIV_INCOMPATIBLE_ROPE_TYPE; - } auto * mem = llama_get_memory(ctx); if (mem == nullptr) { return LLAMA_RS_MEMORY_SEQ_DIV_NULL_MEM; diff --git a/llama-cpp-bindings-sys/wrapper_common.h b/llama-cpp-bindings-sys/wrapper_common.h index 0c09463d3..f69d4dc72 100644 --- a/llama-cpp-bindings-sys/wrapper_common.h +++ b/llama-cpp-bindings-sys/wrapper_common.h @@ -129,8 +129,6 @@ llama_rs_memory_seq_pos_max_status llama_rs_memory_seq_pos_max( typedef enum llama_rs_memory_seq_add_status { LLAMA_RS_MEMORY_SEQ_ADD_OK = 0, LLAMA_RS_MEMORY_SEQ_ADD_NULL_CTX_ARG, - LLAMA_RS_MEMORY_SEQ_ADD_NULL_MODEL, - LLAMA_RS_MEMORY_SEQ_ADD_INCOMPATIBLE_ROPE_TYPE, LLAMA_RS_MEMORY_SEQ_ADD_NULL_MEM, LLAMA_RS_MEMORY_SEQ_ADD_ERROR_STRING_ALLOCATION_FAILED, LLAMA_RS_MEMORY_SEQ_ADD_LLAMA_CPP_OUT_OF_MEMORY, @@ -148,8 +146,6 @@ llama_rs_memory_seq_add_status llama_rs_memory_seq_add( typedef enum llama_rs_memory_seq_div_status { LLAMA_RS_MEMORY_SEQ_DIV_OK = 0, LLAMA_RS_MEMORY_SEQ_DIV_NULL_CTX_ARG, - LLAMA_RS_MEMORY_SEQ_DIV_NULL_MODEL, - LLAMA_RS_MEMORY_SEQ_DIV_INCOMPATIBLE_ROPE_TYPE, LLAMA_RS_MEMORY_SEQ_DIV_NULL_MEM, LLAMA_RS_MEMORY_SEQ_DIV_ERROR_STRING_ALLOCATION_FAILED, LLAMA_RS_MEMORY_SEQ_DIV_LLAMA_CPP_OUT_OF_MEMORY, diff --git a/llama-cpp-bindings/src/context/memory.rs b/llama-cpp-bindings/src/context/memory.rs index ff9640180..7df2fe467 100644 --- a/llama-cpp-bindings/src/context/memory.rs +++ b/llama-cpp-bindings/src/context/memory.rs @@ -14,6 +14,7 @@ use crate::error::memory_seq_cp_error::MemorySeqCpError; use crate::error::memory_seq_div_error::MemorySeqDivError; use crate::error::memory_seq_pos_max_error::MemorySeqPosMaxError; use crate::error::memory_seq_rm_error::MemorySeqRmError; +use crate::model::rope_type::RopeType; fn memory_seq_add_status_to_result( status: llama_cpp_bindings_sys::llama_rs_memory_seq_add_status, @@ -21,9 +22,6 @@ fn memory_seq_add_status_to_result( ) -> Result<(), MemorySeqAddError> { match status { llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_INCOMPATIBLE_ROPE_TYPE => { - Err(MemorySeqAddError::IncompatibleRopeType) - } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_MEM => { Err(MemorySeqAddError::MemoryHandleUnavailable) } @@ -48,11 +46,6 @@ fn memory_seq_add_status_to_result( detail: "was given a null ctx argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_MODEL => Err(FfiContractError { - operation: "llama_rs_memory_seq_add", - detail: "was given a null model argument", - } - .into()), other => Err(FfiStatusError { operation: "llama_rs_memory_seq_add", code: i64::from(other), @@ -67,9 +60,6 @@ fn memory_seq_div_status_to_result( ) -> Result<(), MemorySeqDivError> { match status { llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_INCOMPATIBLE_ROPE_TYPE => { - Err(MemorySeqDivError::IncompatibleRopeType) - } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_MEM => { Err(MemorySeqDivError::MemoryHandleUnavailable) } @@ -94,11 +84,6 @@ fn memory_seq_div_status_to_result( detail: "was given a null ctx argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_MODEL => Err(FfiContractError { - operation: "llama_rs_memory_seq_div", - detail: "was given a null model argument", - } - .into()), other => Err(FfiStatusError { operation: "llama_rs_memory_seq_div", code: i64::from(other), @@ -170,6 +155,12 @@ impl LlamaContext<'_> { memory_of(self.context).is_some() } + fn shifts_positions_one_per_embedding(&self) -> bool { + self.model + .rope_type() + .is_none_or(RopeType::has_one_position_per_embedding) + } + fn required_memory_handle( &self, ) -> Result { @@ -257,6 +248,11 @@ impl LlamaContext<'_> { let p1 = p1 .map_or(Ok(-1), i32::try_from) .map_err(MemorySeqAddError::P1TooLarge)?; + + if !self.shifts_positions_one_per_embedding() { + return Err(MemorySeqAddError::IncompatibleRopeType); + } + let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_memory_seq_add( @@ -286,6 +282,11 @@ impl LlamaContext<'_> { let p1 = p1 .map_or(Ok(-1), i32::try_from) .map_err(MemorySeqDivError::P1TooLarge)?; + + if !self.shifts_positions_one_per_embedding() { + return Err(MemorySeqDivError::IncompatibleRopeType); + } + let divisor = c_int::from(divisor.get()); let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { @@ -341,17 +342,6 @@ mod memory_seq_add_status_to_result_tests { ); } - #[test] - fn add_incompatible_rope_type_status_maps_to_incompatible_rope_type() { - assert_eq!( - memory_seq_add_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_INCOMPATIBLE_ROPE_TYPE, - ptr::null_mut(), - ), - Err(MemorySeqAddError::IncompatibleRopeType) - ); - } - #[test] fn add_null_mem_status_maps_to_memory_handle_unavailable() { assert_eq!( @@ -436,24 +426,6 @@ mod memory_seq_add_status_to_result_tests { ); } - #[test] - fn memory_seq_add_status_to_result_reports_a_null_model_as_a_contract_violation() { - let outcome = memory_seq_add_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_MODEL, - ptr::null_mut(), - ); - assert_eq!( - outcome.err(), - Some( - FfiContractError { - operation: "llama_rs_memory_seq_add", - detail: "was given a null model argument", - } - .into() - ) - ); - } - #[test] fn memory_seq_add_status_to_result_reports_llama_cpp_running_out_of_memory() { let outcome = memory_seq_add_status_to_result( @@ -485,17 +457,6 @@ mod memory_seq_div_status_to_result_tests { ); } - #[test] - fn div_incompatible_rope_type_status_maps_to_incompatible_rope_type() { - assert_eq!( - memory_seq_div_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_INCOMPATIBLE_ROPE_TYPE, - ptr::null_mut(), - ), - Err(MemorySeqDivError::IncompatibleRopeType) - ); - } - #[test] fn div_null_mem_status_maps_to_memory_handle_unavailable() { assert_eq!( @@ -580,24 +541,6 @@ mod memory_seq_div_status_to_result_tests { ); } - #[test] - fn memory_seq_div_status_to_result_reports_a_null_model_as_a_contract_violation() { - let outcome = memory_seq_div_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_MODEL, - ptr::null_mut(), - ); - assert_eq!( - outcome.err(), - Some( - FfiContractError { - operation: "llama_rs_memory_seq_div", - detail: "was given a null model argument", - } - .into() - ) - ); - } - #[test] fn memory_seq_div_status_to_result_reports_llama_cpp_running_out_of_memory() { let outcome = memory_seq_div_status_to_result( diff --git a/llama-cpp-bindings/src/model/rope_type.rs b/llama-cpp-bindings/src/model/rope_type.rs index 2a970918b..53b7af021 100644 --- a/llama-cpp-bindings/src/model/rope_type.rs +++ b/llama-cpp-bindings/src/model/rope_type.rs @@ -19,6 +19,16 @@ impl RopeType { _ => None, } } + + /// Whether every embedding carries a single position, which llama.cpp requires before it + /// shifts or divides the positions of a sequence in memory. + #[must_use] + pub const fn has_one_position_per_embedding(self) -> bool { + match self { + Self::Norm | Self::NeoX | Self::Vision => true, + Self::MRope | Self::IMRope => false, + } + } } #[cfg(test)] @@ -49,4 +59,19 @@ mod tests { ] ); } + + #[test] + fn only_multi_section_rope_types_place_several_positions_per_embedding() { + assert_eq!( + [ + RopeType::Norm, + RopeType::NeoX, + RopeType::MRope, + RopeType::IMRope, + RopeType::Vision, + ] + .map(RopeType::has_one_position_per_embedding), + [true, true, false, false, true] + ); + } } From 00065815854ac166ccf7fb837d15f1bb345e5ad5 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 07:35:19 +0100 Subject: [PATCH 47/82] Drop chat message statuses typed messages cannot trigger and refuse unrecognized wrapper enum values --- .../wrapper_chat_session.cpp | 16 ++----- llama-cpp-bindings-sys/wrapper_chat_session.h | 2 - .../tests/chat_session_input_validation.rs | 43 +++++++++++++++++++ llama-cpp-bindings/src/chat_session.rs | 22 +--------- .../src/error/chat_session_error.rs | 4 -- 5 files changed, 47 insertions(+), 40 deletions(-) diff --git a/llama-cpp-bindings-sys/wrapper_chat_session.cpp b/llama-cpp-bindings-sys/wrapper_chat_session.cpp index 5373a27e1..abe55c4d8 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_session.cpp +++ b/llama-cpp-bindings-sys/wrapper_chat_session.cpp @@ -53,7 +53,7 @@ auto tool_choice_of(llama_rs_chat_tool_choice tool_choice) -> common_chat_tool_c return COMMON_CHAT_TOOL_CHOICE_AUTO; } - return COMMON_CHAT_TOOL_CHOICE_AUTO; + throw std::invalid_argument("unrecognized llama_rs_chat_tool_choice value"); } auto continuation_of(llama_rs_chat_assistant_turn assistant_turn) -> common_chat_continuation { @@ -67,7 +67,7 @@ auto continuation_of(llama_rs_chat_assistant_turn assistant_turn) -> common_chat return COMMON_CHAT_CONTINUATION_NONE; } - return COMMON_CHAT_CONTINUATION_NONE; + throw std::invalid_argument("unrecognized llama_rs_chat_assistant_turn value"); } auto span_kind_of_tag(const std::string & tag) -> std::optional { @@ -179,17 +179,7 @@ extern "C" auto llama_rs_chat_session_create( try { common_chat_templates_inputs inputs; - try { - inputs.messages = common_chat_msgs_parse_oaicompat(common_json::parse(messages_json)); - } catch (const common_json_error & err) { - return llama_rs_report_exception_message( - out_error, err.what(), LLAMA_RS_CHAT_SESSION_CREATE_MESSAGES_NOT_JSON, - LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED); - } catch (const std::invalid_argument & err) { - return llama_rs_report_exception_message( - out_error, err.what(), LLAMA_RS_CHAT_SESSION_CREATE_MESSAGES_INVALID, - LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED); - } + inputs.messages = common_chat_msgs_parse_oaicompat(common_json::parse(messages_json)); try { inputs.tools = common_chat_tools_parse_oaicompat(common_json::parse(tools_json)); diff --git a/llama-cpp-bindings-sys/wrapper_chat_session.h b/llama-cpp-bindings-sys/wrapper_chat_session.h index 4cbeedc3d..59550e8dc 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_session.h +++ b/llama-cpp-bindings-sys/wrapper_chat_session.h @@ -46,8 +46,6 @@ typedef enum llama_rs_chat_session_create_status { LLAMA_RS_CHAT_SESSION_CREATE_NULL_TOOLS_JSON_ARG, LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_SESSION_ARG, LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_ERROR_ARG, - LLAMA_RS_CHAT_SESSION_CREATE_MESSAGES_NOT_JSON, - LLAMA_RS_CHAT_SESSION_CREATE_MESSAGES_INVALID, LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_NOT_JSON, LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_INVALID, LLAMA_RS_CHAT_SESSION_CREATE_TEMPLATE_CANNOT_SERVE_REQUEST, diff --git a/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs index d4810e82c..92e50aeb6 100644 --- a/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs +++ b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs @@ -166,3 +166,46 @@ fn tokenizing_a_prompt_whose_reasoning_prefill_holds_a_nul_character_is_refused( Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn a_conversation_the_template_raises_on_is_refused_with_the_template_message( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let result = ChatSession::new( + fixture.model, + fixture.model.chat_templates()?, + &[ + LlamaChatMessage::new("user".to_owned(), "hello".to_owned())?, + LlamaChatMessage::new("user".to_owned(), "are you there?".to_owned())?, + ], + "[]", + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::New, + enable_thinking: false, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + ); + + assert_eq!( + result.err(), + Some(ChatSessionError::TemplateCannotServeRequest { + message: "Unable to generate parser for this template. Automatic parser generation \ + failed: \n------------\nWhile executing CallExpression at line 19, column \ + 27 in source:\n...% 2 == 0) -%}↵ {{ raise_exception(\"Conversation \ + roles must alternate user...\n \ + ^\nError: Jinja Exception: Conversation roles must alternate \ + user/assistant/user/assistant/..." + .to_owned() + }) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings/src/chat_session.rs b/llama-cpp-bindings/src/chat_session.rs index d8c8c3e21..ac2a8951a 100644 --- a/llama-cpp-bindings/src/chat_session.rs +++ b/llama-cpp-bindings/src/chat_session.rs @@ -111,16 +111,6 @@ fn chat_session_create_status_to_result( .ok_or_else(|| { contract_violation(OPERATION, "success status contained a null session").into() }), - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_MESSAGES_NOT_JSON => { - Err(ChatSessionError::MessagesNotJson { - message: reported_message()?, - }) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_MESSAGES_INVALID => { - Err(ChatSessionError::MessagesInvalid { - message: reported_message()?, - }) - } llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_NOT_JSON => { Err(ChatSessionError::ToolsNotJson { message: reported_message()?, @@ -779,15 +769,7 @@ mod tests { #[test] fn creating_a_session_reports_why_llama_cpp_refused_the_request() { - let refusals: [CreateRefusal; 6] = [ - ( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_MESSAGES_NOT_JSON, - |message| ChatSessionError::MessagesNotJson { message }, - ), - ( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_MESSAGES_INVALID, - |message| ChatSessionError::MessagesInvalid { message }, - ), + let refusals: [CreateRefusal; 4] = [ ( llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_NOT_JSON, |message| ChatSessionError::ToolsNotJson { message }, @@ -820,8 +802,6 @@ mod tests { #[test] fn creating_a_session_reports_a_refusal_without_a_message_as_a_contract_violation() { let refusals = [ - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_MESSAGES_NOT_JSON, - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_MESSAGES_INVALID, llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_NOT_JSON, llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_INVALID, llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TEMPLATE_CANNOT_SERVE_REQUEST, diff --git a/llama-cpp-bindings/src/error/chat_session_error.rs b/llama-cpp-bindings/src/error/chat_session_error.rs index 2221510da..04d179703 100644 --- a/llama-cpp-bindings/src/error/chat_session_error.rs +++ b/llama-cpp-bindings/src/error/chat_session_error.rs @@ -21,10 +21,6 @@ pub enum ChatSessionError { MessageNotUtf8(#[source] Utf8Error), #[error("the chat request contains an interior NUL byte")] RequestContainsNul(#[source] NulError), - #[error("the messages are not valid JSON: {message}")] - MessagesNotJson { message: String }, - #[error("the messages are not valid chat messages: {message}")] - MessagesInvalid { message: String }, #[error("the tools are not valid JSON: {message}")] ToolsNotJson { message: String }, #[error("the tools are not valid tool definitions: {message}")] From f04c51988080c958f1b53c8c3d59f9478a2e1ab9 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 07:36:06 +0100 Subject: [PATCH 48/82] Decode a token llama.cpp renders as nothing into an empty piece instead of an error --- .../tests/detokenization.rs | 23 +++++++++++++++++++ .../src/error/token_to_string_error.rs | 2 -- llama-cpp-bindings/src/model.rs | 16 ++----------- 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/llama-cpp-bindings-tests/tests/detokenization.rs b/llama-cpp-bindings-tests/tests/detokenization.rs index 6529b28f2..084e47cd2 100644 --- a/llama-cpp-bindings-tests/tests/detokenization.rs +++ b/llama-cpp-bindings-tests/tests/detokenization.rs @@ -343,3 +343,26 @@ fn token_to_piece_replaces_a_lone_continuation_byte_with_the_replacement_charact Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn a_control_token_decoded_without_special_rendering_is_an_empty_piece(fixture: &LlamaFixture<'_>) { + let model = fixture.model; + let control_token = single_token(model, "<|im_start|>"); + + assert_eq!( + model.token_to_piece( + &SampledToken::Content(control_token), + &mut encoding_rs::UTF_8.new_decoder(), + false, + None + ), + Ok(String::new()) + ); +} diff --git a/llama-cpp-bindings/src/error/token_to_string_error.rs b/llama-cpp-bindings/src/error/token_to_string_error.rs index 7072618ea..6e572ce5e 100644 --- a/llama-cpp-bindings/src/error/token_to_string_error.rs +++ b/llama-cpp-bindings/src/error/token_to_string_error.rs @@ -8,8 +8,6 @@ use crate::error::token_outside_vocabulary::TokenOutsideVocabulary; pub enum TokenToStringError { #[error(transparent)] TokenOutsideVocabulary(#[from] TokenOutsideVocabulary), - #[error("Unknown Token Type")] - UnknownTokenType, #[error("Insufficient Buffer Space {0}")] InsufficientBufferSpace(c_int), #[error("FromUtf8Error {0}")] diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index a02528c9c..6388627d4 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -409,7 +409,6 @@ impl LlamaModel { }; match size { - 0 => Err(TokenToStringError::UnknownTokenType), error_code if error_code.is_negative() => { Err(TokenToStringError::InsufficientBufferSpace(error_code)) } @@ -691,17 +690,6 @@ impl LlamaModel { } } -fn token_piece_bytes_for_tok_env( - model: &LlamaModel, - token: LlamaToken, - special: bool, -) -> Result, TokenToStringError> { - match model.read_piece(token, special, None, <[u8]>::to_vec) { - Err(TokenToStringError::UnknownTokenType) => Ok(Vec::new()), - piece_result => piece_result, - } -} - fn build_approximate_tok_env( model: &LlamaModel, ) -> Result, TokenToStringError> { @@ -720,9 +708,9 @@ fn build_approximate_tok_env( for token_id in 0..n_vocab.cast_signed() { let token = LlamaToken(token_id); - let bytes = token_piece_bytes_for_tok_env(model, token, false)?; + let bytes = model.read_piece(token, false, None, <[u8]>::to_vec)?; if bytes.is_empty() { - let special_bytes = token_piece_bytes_for_tok_env(model, token, true)?; + let special_bytes = model.read_piece(token, true, None, <[u8]>::to_vec)?; if special_bytes.is_empty() { words.push(vec![]); } else { From 29cdbd08f2bf9257dd8dfb03c056e765200b0da6 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 07:38:52 +0100 Subject: [PATCH 49/82] Refuse memory edits on sequence ids the context does not have --- llama-cpp-bindings-sys/wrapper_common.cpp | 4 - llama-cpp-bindings-sys/wrapper_common.h | 1 - .../tests/memory_availability.rs | 2 +- .../tests/memory_occupancy.rs | 2 +- .../tests/memory_position_bounds.rs | 5 +- .../tests/memory_sequence_edits.rs | 4 +- .../tests/memory_sequence_id_bounds.rs | 139 +++++++++++++++++- llama-cpp-bindings/src/context/memory.rs | 77 +++++----- .../src/error/memory_seq_add_error.rs | 4 + .../src/error/memory_seq_cp_error.rs | 4 + .../src/error/memory_seq_div_error.rs | 4 + .../src/error/memory_seq_pos_max_error.rs | 6 +- .../src/error/memory_seq_rm_error.rs | 4 + 13 files changed, 202 insertions(+), 54 deletions(-) diff --git a/llama-cpp-bindings-sys/wrapper_common.cpp b/llama-cpp-bindings-sys/wrapper_common.cpp index 75c93f116..92662af3f 100644 --- a/llama-cpp-bindings-sys/wrapper_common.cpp +++ b/llama-cpp-bindings-sys/wrapper_common.cpp @@ -187,10 +187,6 @@ extern "C" auto llama_rs_memory_seq_pos_max( if (mem == nullptr) { return LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_MEM; } - uint32_t const n_seq_max = llama_n_seq_max(ctx); - if (seq_id < 0 || (uint32_t) seq_id >= n_seq_max) { - return LLAMA_RS_MEMORY_SEQ_POS_MAX_SEQ_ID_OUT_OF_RANGE; - } *out_position = llama_memory_seq_pos_max(mem, seq_id); return LLAMA_RS_MEMORY_SEQ_POS_MAX_OK; diff --git a/llama-cpp-bindings-sys/wrapper_common.h b/llama-cpp-bindings-sys/wrapper_common.h index f69d4dc72..b2cdaa7ac 100644 --- a/llama-cpp-bindings-sys/wrapper_common.h +++ b/llama-cpp-bindings-sys/wrapper_common.h @@ -114,7 +114,6 @@ typedef enum llama_rs_memory_seq_pos_max_status { LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_OUT_POSITION_ARG, LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_OUT_ERROR_ARG, LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_MEM, - LLAMA_RS_MEMORY_SEQ_POS_MAX_SEQ_ID_OUT_OF_RANGE, LLAMA_RS_MEMORY_SEQ_POS_MAX_ERROR_STRING_ALLOCATION_FAILED, LLAMA_RS_MEMORY_SEQ_POS_MAX_LLAMA_CPP_OUT_OF_MEMORY, LLAMA_RS_MEMORY_SEQ_POS_MAX_LLAMA_CPP_THREW_CXX_EXCEPTION, diff --git a/llama-cpp-bindings-tests/tests/memory_availability.rs b/llama-cpp-bindings-tests/tests/memory_availability.rs index c83d28964..b643162ea 100644 --- a/llama-cpp-bindings-tests/tests/memory_availability.rs +++ b/llama-cpp-bindings-tests/tests/memory_availability.rs @@ -57,8 +57,8 @@ fn memory_edits_without_a_memory_module_trivially_succeed( let mut context = fixture.build_context()?; context.memory_clear(); - context.memory_seq_keep(0); + assert_eq!(context.memory_seq_keep(0), Ok(())); assert_eq!(context.memory_seq_rm(Some(0), None, None), Ok(())); Ok(()) diff --git a/llama-cpp-bindings-tests/tests/memory_occupancy.rs b/llama-cpp-bindings-tests/tests/memory_occupancy.rs index 5da38d551..330255a32 100644 --- a/llama-cpp-bindings-tests/tests/memory_occupancy.rs +++ b/llama-cpp-bindings-tests/tests/memory_occupancy.rs @@ -144,7 +144,7 @@ fn memory_seq_keep_removes_every_other_sequence(fixture: &LlamaFixture<'_>) -> R let primed_position_max = context.memory_seq_pos_max(0)?; context.memory_seq_cp(0, 1, None, None)?; - context.memory_seq_keep(1); + context.memory_seq_keep(1)?; assert_eq!(context.memory_seq_pos_max(0)?, -1); assert_eq!(context.memory_seq_pos_max(1)?, primed_position_max); diff --git a/llama-cpp-bindings-tests/tests/memory_position_bounds.rs b/llama-cpp-bindings-tests/tests/memory_position_bounds.rs index 0daf9c4c5..28459d42a 100644 --- a/llama-cpp-bindings-tests/tests/memory_position_bounds.rs +++ b/llama-cpp-bindings-tests/tests/memory_position_bounds.rs @@ -7,6 +7,7 @@ use llama_cpp_bindings::error::memory_seq_add_error::MemorySeqAddError; use llama_cpp_bindings::error::memory_seq_cp_error::MemorySeqCpError; use llama_cpp_bindings::error::memory_seq_div_error::MemorySeqDivError; use llama_cpp_bindings::error::memory_seq_rm_error::MemorySeqRmError; +use llama_cpp_bindings_tests::two_sequence_context::two_sequence_context; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -43,7 +44,7 @@ use llama_cpp_test_harness_macros::llama_test; n_ubatch = 128, )] fn memory_seq_cp_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; + let mut context = two_sequence_context(fixture)?; let result = context.memory_seq_cp(0, 1, Some(u32::MAX), None); @@ -90,7 +91,7 @@ fn memory_seq_cp_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Res n_ubatch = 128, )] fn memory_seq_cp_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; + let mut context = two_sequence_context(fixture)?; let result = context.memory_seq_cp(0, 1, Some(0), Some(u32::MAX)); diff --git a/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs b/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs index 0c9429286..c84a07d45 100644 --- a/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs +++ b/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs @@ -127,11 +127,11 @@ fn memory_seq_cp_copies_a_whole_primed_sequence(fixture: &LlamaFixture<'_>) -> R n_ubatch = 128, )] fn memory_seq_cp_accepts_an_explicit_range(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; + let mut context = two_sequence_context(fixture)?; prime_memory(fixture, &mut context)?; - assert_eq!(context.memory_seq_cp(0, 2, Some(0), Some(1)), Ok(())); + assert_eq!(context.memory_seq_cp(0, 1, Some(0), Some(1)), Ok(())); Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs b/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs index e734e94c1..d40f25521 100644 --- a/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs +++ b/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs @@ -1,7 +1,13 @@ +use std::num::NonZeroU8; + use anyhow::Result; +use llama_cpp_bindings::error::memory_seq_add_error::MemorySeqAddError; +use llama_cpp_bindings::error::memory_seq_cp_error::MemorySeqCpError; +use llama_cpp_bindings::error::memory_seq_div_error::MemorySeqDivError; use llama_cpp_bindings::error::memory_seq_pos_max_error::MemorySeqPosMaxError; use llama_cpp_bindings::error::memory_seq_rm_error::MemorySeqRmError; +use llama_cpp_bindings::error::sequence_id_out_of_range::SequenceIdOutOfRange; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -44,7 +50,12 @@ fn memory_seq_pos_max_rejects_out_of_range_sequence(fixture: &LlamaFixture<'_>) assert_eq!( result, - Err(MemorySeqPosMaxError::SequenceIdOutOfRange { seq_id: 999 }) + Err(MemorySeqPosMaxError::SequenceIdOutOfRange( + SequenceIdOutOfRange { + seq_id: 999, + n_seq_max: 1 + } + )) ); Ok(()) @@ -96,3 +107,129 @@ fn memory_seq_rm_rejects_sequence_id_exceeding_i32_max(fixture: &LlamaFixture<'_ Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_cp_rejects_a_destination_outside_the_context_sequences( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + assert_eq!( + context.memory_seq_cp(0, 1, None, None), + Err(MemorySeqCpError::SequenceIdOutOfRange( + SequenceIdOutOfRange { + seq_id: 1, + n_seq_max: 1 + } + )) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_keep_rejects_a_sequence_outside_the_context_sequences( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + assert_eq!( + context.memory_seq_keep(1), + Err(SequenceIdOutOfRange { + seq_id: 1, + n_seq_max: 1 + }) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_add_rejects_a_negative_sequence_id(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + assert_eq!( + context.memory_seq_add(-1, None, None, 1), + Err(MemorySeqAddError::SequenceIdOutOfRange( + SequenceIdOutOfRange { + seq_id: -1, + n_seq_max: 1 + } + )) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_div_rejects_a_sequence_outside_the_context_sequences( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + assert_eq!( + context.memory_seq_div(1, None, None, NonZeroU8::MIN), + Err(MemorySeqDivError::SequenceIdOutOfRange( + SequenceIdOutOfRange { + seq_id: 1, + n_seq_max: 1 + } + )) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_rm_rejects_a_sequence_outside_the_context_sequences( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + assert_eq!( + context.memory_seq_rm(Some(1), None, None), + Err(MemorySeqRmError::SequenceIdOutOfRange( + SequenceIdOutOfRange { + seq_id: 1, + n_seq_max: 1 + } + )) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings/src/context/memory.rs b/llama-cpp-bindings/src/context/memory.rs index 7df2fe467..c650edc86 100644 --- a/llama-cpp-bindings/src/context/memory.rs +++ b/llama-cpp-bindings/src/context/memory.rs @@ -14,6 +14,7 @@ use crate::error::memory_seq_cp_error::MemorySeqCpError; use crate::error::memory_seq_div_error::MemorySeqDivError; use crate::error::memory_seq_pos_max_error::MemorySeqPosMaxError; use crate::error::memory_seq_rm_error::MemorySeqRmError; +use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; use crate::model::rope_type::RopeType; fn memory_seq_add_status_to_result( @@ -95,7 +96,6 @@ fn memory_seq_div_status_to_result( fn memory_seq_pos_max_status_to_result( status: llama_cpp_bindings_sys::llama_rs_memory_seq_pos_max_status, position: i32, - seq_id: i32, out_error: *mut c_char, ) -> Result { match status { @@ -122,9 +122,6 @@ fn memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_MEM => { Err(MemorySeqPosMaxError::MemoryHandleUnavailable) } - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_SEQ_ID_OUT_OF_RANGE => { - Err(MemorySeqPosMaxError::SequenceIdOutOfRange { seq_id }) - } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_ERROR_STRING_ALLOCATION_FAILED => { Err(MemorySeqPosMaxError::NotEnoughMemory) } @@ -168,7 +165,8 @@ impl LlamaContext<'_> { } /// # Errors - /// If either position exceeds [`i32::MAX`], or the context has no memory module. + /// If either sequence id names no sequence of the context, either position exceeds + /// [`i32::MAX`], or the context has no memory module. pub fn memory_seq_cp( &mut self, src: i32, @@ -176,6 +174,8 @@ impl LlamaContext<'_> { p0: Option, p1: Option, ) -> Result<(), MemorySeqCpError> { + self.limits.ensure_sequence_id(src)?; + self.limits.ensure_sequence_id(dest)?; let p0 = p0 .map_or(Ok(-1), i32::try_from) .map_err(MemorySeqCpError::P0TooLarge)?; @@ -189,9 +189,10 @@ impl LlamaContext<'_> { } /// # Errors - /// If the sequence id or either position exceeds [`i32::MAX`], or llama.cpp reports - /// that the partial sequence could not be removed. A context without a memory module - /// holds no KV cache, so the removal trivially succeeds. + /// If the sequence id or either position exceeds [`i32::MAX`], the sequence id names no + /// sequence of the context, or llama.cpp reports that the partial sequence could not be + /// removed. `None` removes from every sequence. A context without a memory module holds no + /// KV cache, so the removal trivially succeeds. pub fn memory_seq_rm( &mut self, seq_id: Option, @@ -199,8 +200,15 @@ impl LlamaContext<'_> { p1: Option, ) -> Result<(), MemorySeqRmError> { let seq_id = seq_id - .map_or(Ok(-1), i32::try_from) + .map(i32::try_from) + .transpose() .map_err(MemorySeqRmError::SeqIdTooLarge)?; + + if let Some(sequence) = seq_id { + self.limits.ensure_sequence_id(sequence)?; + } + + let seq_id = seq_id.unwrap_or(-1); let p0 = p0 .map_or(Ok(-1), i32::try_from) .map_err(MemorySeqRmError::P0TooLarge)?; @@ -226,15 +234,21 @@ impl LlamaContext<'_> { unsafe { llama_cpp_bindings_sys::llama_memory_clear(mem, clear_data_buffers) }; } - pub fn memory_seq_keep(&mut self, seq_id: i32) { - let Some(mem) = memory_of(self.context) else { - return; - }; - unsafe { llama_cpp_bindings_sys::llama_memory_seq_keep(mem, seq_id) }; + /// # Errors + /// If the sequence id names no sequence of the context. + pub fn memory_seq_keep(&mut self, seq_id: i32) -> Result<(), SequenceIdOutOfRange> { + self.limits.ensure_sequence_id(seq_id)?; + + if let Some(mem) = memory_of(self.context) { + unsafe { llama_cpp_bindings_sys::llama_memory_seq_keep(mem, seq_id) }; + } + + Ok(()) } /// # Errors - /// If either position exceeds [`i32::MAX`], or the underlying memory operation reports a failure. + /// If the sequence id names no sequence of the context, either position exceeds + /// [`i32::MAX`], or the underlying memory operation reports a failure. pub fn memory_seq_add( &mut self, seq_id: i32, @@ -242,6 +256,7 @@ impl LlamaContext<'_> { p1: Option, delta: i32, ) -> Result<(), MemorySeqAddError> { + self.limits.ensure_sequence_id(seq_id)?; let p0 = p0 .map_or(Ok(-1), i32::try_from) .map_err(MemorySeqAddError::P0TooLarge)?; @@ -268,7 +283,8 @@ impl LlamaContext<'_> { } /// # Errors - /// If either position exceeds [`i32::MAX`], or the underlying memory operation reports a failure. + /// If the sequence id names no sequence of the context, either position exceeds + /// [`i32::MAX`], or the underlying memory operation reports a failure. pub fn memory_seq_div( &mut self, seq_id: i32, @@ -276,6 +292,7 @@ impl LlamaContext<'_> { p1: Option, divisor: NonZeroU8, ) -> Result<(), MemorySeqDivError> { + self.limits.ensure_sequence_id(seq_id)?; let p0 = p0 .map_or(Ok(-1), i32::try_from) .map_err(MemorySeqDivError::P0TooLarge)?; @@ -306,6 +323,8 @@ impl LlamaContext<'_> { /// /// Returns [`MemorySeqPosMaxError`] if the sequence does not exist or the memory lookup fails. pub fn memory_seq_pos_max(&self, seq_id: i32) -> Result { + self.limits.ensure_sequence_id(seq_id)?; + let mut position = -1; let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { @@ -317,7 +336,7 @@ impl LlamaContext<'_> { ) }; - memory_seq_pos_max_status_to_result(status, position, seq_id, out_error) + memory_seq_pos_max_status_to_result(status, position, out_error) } } @@ -567,7 +586,6 @@ mod memory_seq_pos_max_status_to_result_tests { memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_OK, 17, - 2, ptr::null_mut(), ), Ok(17) @@ -580,7 +598,6 @@ mod memory_seq_pos_max_status_to_result_tests { memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_CTX_ARG, -1, - 2, ptr::null_mut(), ), Err(MemorySeqPosMaxError::FfiContract(FfiContractError { @@ -596,7 +613,6 @@ mod memory_seq_pos_max_status_to_result_tests { memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_OUT_POSITION_ARG, -1, - 2, ptr::null_mut(), ), Err(MemorySeqPosMaxError::FfiContract(FfiContractError { @@ -612,7 +628,6 @@ mod memory_seq_pos_max_status_to_result_tests { memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_OUT_ERROR_ARG, -1, - 2, ptr::null_mut(), ), Err(MemorySeqPosMaxError::FfiContract(FfiContractError { @@ -628,33 +643,18 @@ mod memory_seq_pos_max_status_to_result_tests { memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_MEM, -1, - 2, ptr::null_mut(), ), Err(MemorySeqPosMaxError::MemoryHandleUnavailable) ); } - #[test] - fn seq_pos_max_out_of_range_status_preserves_sequence_id() { - assert_eq!( - memory_seq_pos_max_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_SEQ_ID_OUT_OF_RANGE, - -1, - 27, - ptr::null_mut(), - ), - Err(MemorySeqPosMaxError::SequenceIdOutOfRange { seq_id: 27 }) - ); - } - #[test] fn seq_pos_max_allocation_failed_status_maps_to_not_enough_memory() { assert_eq!( memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_ERROR_STRING_ALLOCATION_FAILED, -1, - 2, ptr::null_mut(), ), Err(MemorySeqPosMaxError::NotEnoughMemory) @@ -667,7 +667,6 @@ mod memory_seq_pos_max_status_to_result_tests { memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_LLAMA_CPP_THREW_CXX_EXCEPTION, -1, - 2, ptr::null_mut(), ), Err(FfiContractError { @@ -688,7 +687,6 @@ mod memory_seq_pos_max_status_to_result_tests { memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_LLAMA_CPP_THREW_CXX_EXCEPTION, -1, - 2, out_error, ), Err(MemorySeqPosMaxError::Reported { @@ -700,7 +698,7 @@ mod memory_seq_pos_max_status_to_result_tests { #[test] fn seq_pos_max_unknown_status_is_preserved() { assert_eq!( - memory_seq_pos_max_status_to_result(255, -1, 2, ptr::null_mut(),), + memory_seq_pos_max_status_to_result(255, -1, ptr::null_mut(),), Err(MemorySeqPosMaxError::FfiStatus(FfiStatusError { operation: "llama_rs_memory_seq_pos_max", code: 255, @@ -713,7 +711,6 @@ mod memory_seq_pos_max_status_to_result_tests { let outcome = memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_LLAMA_CPP_OUT_OF_MEMORY, 0, - 0, ptr::null_mut(), ); assert_eq!( diff --git a/llama-cpp-bindings/src/error/memory_seq_add_error.rs b/llama-cpp-bindings/src/error/memory_seq_add_error.rs index ca936c92d..b93298834 100644 --- a/llama-cpp-bindings/src/error/memory_seq_add_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_add_error.rs @@ -3,6 +3,8 @@ use std::num::TryFromIntError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum MemorySeqAddError { #[error(transparent)] @@ -17,6 +19,8 @@ pub enum MemorySeqAddError { IncompatibleRopeType, #[error("context has no memory module available")] MemoryHandleUnavailable, + #[error(transparent)] + SequenceIdOutOfRange(#[from] SequenceIdOutOfRange), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/error/memory_seq_cp_error.rs b/llama-cpp-bindings/src/error/memory_seq_cp_error.rs index 37e4a15b9..863ffa4e4 100644 --- a/llama-cpp-bindings/src/error/memory_seq_cp_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_cp_error.rs @@ -1,5 +1,7 @@ use std::num::TryFromIntError; +use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum MemorySeqCpError { #[error("provided start position is too large for an i32")] @@ -8,4 +10,6 @@ pub enum MemorySeqCpError { P1TooLarge(#[source] TryFromIntError), #[error("the context has no memory module attached")] MemoryHandleUnavailable, + #[error(transparent)] + SequenceIdOutOfRange(#[from] SequenceIdOutOfRange), } diff --git a/llama-cpp-bindings/src/error/memory_seq_div_error.rs b/llama-cpp-bindings/src/error/memory_seq_div_error.rs index 87f26b9b3..d428d6859 100644 --- a/llama-cpp-bindings/src/error/memory_seq_div_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_div_error.rs @@ -3,6 +3,8 @@ use std::num::TryFromIntError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum MemorySeqDivError { #[error(transparent)] @@ -17,6 +19,8 @@ pub enum MemorySeqDivError { IncompatibleRopeType, #[error("context has no memory module available")] MemoryHandleUnavailable, + #[error(transparent)] + SequenceIdOutOfRange(#[from] SequenceIdOutOfRange), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs b/llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs index 795d02744..7bf1601c1 100644 --- a/llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs @@ -1,6 +1,8 @@ use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum MemorySeqPosMaxError { #[error(transparent)] @@ -9,8 +11,8 @@ pub enum MemorySeqPosMaxError { FfiStatus(#[from] FfiStatusError), #[error("context has no memory module available")] MemoryHandleUnavailable, - #[error("sequence id {seq_id} is outside the context sequence range")] - SequenceIdOutOfRange { seq_id: i32 }, + #[error(transparent)] + SequenceIdOutOfRange(#[from] SequenceIdOutOfRange), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/error/memory_seq_rm_error.rs b/llama-cpp-bindings/src/error/memory_seq_rm_error.rs index be9b70b8e..22560d2e7 100644 --- a/llama-cpp-bindings/src/error/memory_seq_rm_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_rm_error.rs @@ -1,6 +1,8 @@ use std::ffi::c_int; use std::num::TryFromIntError; +use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum MemorySeqRmError { #[error("provided sequence id is too large for an i32")] @@ -11,4 +13,6 @@ pub enum MemorySeqRmError { P1TooLarge(#[source] TryFromIntError), #[error("sequence {seq_id} could not be partially removed over positions [{p0}, {p1})")] PartialSequenceNotRemoved { seq_id: c_int, p0: c_int, p1: c_int }, + #[error(transparent)] + SequenceIdOutOfRange(#[from] SequenceIdOutOfRange), } From d9a046974abf51551d7ab194de5c155ea70fab30 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 07:45:15 +0100 Subject: [PATCH 50/82] Map every llama.cpp and ggml enum value to its own variant and refuse values they do not define --- llama-cpp-bindings-sys/wrapper_common.cpp | 16 +++ llama-cpp-bindings-sys/wrapper_common.h | 4 + llama-cpp-bindings-sys/wrapper_context.cpp | 16 +++ llama-cpp-bindings-sys/wrapper_context.h | 4 + .../wrapper_fault_injection.h | 4 + .../tests/backend_device_listing.rs | 59 ++++++++ .../tests/context_creation_limits.rs | 43 ++++++ llama-cpp-bindings-tests/tests/main.rs | 1 + ...emory_position_shift_rope_compatibility.rs | 50 +++++++ .../tests/model_architecture.rs | 17 +-- llama-cpp-bindings/src/context.rs | 11 +- .../src/context/llama_attention_type.rs | 77 +++++++---- .../src/context/llama_pooling_type.rs | 102 ++++++++------ llama-cpp-bindings/src/context/memory.rs | 9 +- llama-cpp-bindings/src/context/params.rs | 41 ++++-- .../src/context/rope_scaling_type.rs | 95 ++++++++----- llama-cpp-bindings/src/error.rs | 7 + .../src/error/backend_device_text_error.rs | 9 ++ .../src/error/llama_backend_device_error.rs | 14 ++ .../src/error/llama_context_load_error.rs | 7 + .../src/error/memory_seq_add_error.rs | 3 + .../src/error/memory_seq_div_error.rs | 3 + .../src/error/unknown_attention_type.rs | 3 + .../src/error/unknown_backend_device_type.rs | 3 + .../src/error/unknown_pooling_type.rs | 3 + .../src/error/unknown_rope_scaling_type.rs | 3 + .../src/error/unknown_rope_type.rs | 3 + llama-cpp-bindings/src/llama_backend.rs | 11 +- .../src/llama_backend_device.rs | 130 ++++++++++-------- .../src/llama_backend_device_type.rs | 68 +++++---- llama-cpp-bindings/src/model.rs | 13 +- llama-cpp-bindings/src/model/rope_type.rs | 58 ++++---- 32 files changed, 634 insertions(+), 253 deletions(-) create mode 100644 llama-cpp-bindings-tests/tests/backend_device_listing.rs create mode 100644 llama-cpp-bindings/src/error/backend_device_text_error.rs create mode 100644 llama-cpp-bindings/src/error/llama_backend_device_error.rs create mode 100644 llama-cpp-bindings/src/error/unknown_attention_type.rs create mode 100644 llama-cpp-bindings/src/error/unknown_backend_device_type.rs create mode 100644 llama-cpp-bindings/src/error/unknown_pooling_type.rs create mode 100644 llama-cpp-bindings/src/error/unknown_rope_scaling_type.rs create mode 100644 llama-cpp-bindings/src/error/unknown_rope_type.rs diff --git a/llama-cpp-bindings-sys/wrapper_common.cpp b/llama-cpp-bindings-sys/wrapper_common.cpp index 92662af3f..fc0c1f886 100644 --- a/llama-cpp-bindings-sys/wrapper_common.cpp +++ b/llama-cpp-bindings-sys/wrapper_common.cpp @@ -507,3 +507,19 @@ extern "C" auto llama_rs_token_to_piece( return llama_token_to_piece(vocab, token, buf, length, lstrip, special); } + +extern "C" auto llama_rs_backend_device_text(const char * text) -> const char * { + if (llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_BACKEND_DEVICE_TEXT_NULL)) { + return nullptr; + } + + return text; +} + +extern "C" auto llama_rs_backend_device_type(ggml_backend_dev_t dev) -> uint32_t { + if (llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_BACKEND_DEVICE_TYPE_UNKNOWN)) { + return GGML_BACKEND_DEVICE_TYPE_META + 1; + } + + return ggml_backend_dev_type(dev); +} diff --git a/llama-cpp-bindings-sys/wrapper_common.h b/llama-cpp-bindings-sys/wrapper_common.h index b2cdaa7ac..64c1972e3 100644 --- a/llama-cpp-bindings-sys/wrapper_common.h +++ b/llama-cpp-bindings-sys/wrapper_common.h @@ -257,6 +257,10 @@ int32_t llama_rs_token_to_piece( int32_t lstrip, bool special); +const char * llama_rs_backend_device_text(const char * text); + +uint32_t llama_rs_backend_device_type(ggml_backend_dev_t dev); + #ifdef __cplusplus } #endif diff --git a/llama-cpp-bindings-sys/wrapper_context.cpp b/llama-cpp-bindings-sys/wrapper_context.cpp index 7b8d7d363..e97589653 100644 --- a/llama-cpp-bindings-sys/wrapper_context.cpp +++ b/llama-cpp-bindings-sys/wrapper_context.cpp @@ -11,6 +11,22 @@ extern "C" auto llama_rs_model_causal_attn(const struct llama_model * model) -> return model->hparams.causal_attn; } +extern "C" auto llama_rs_pooling_type(const struct llama_context * ctx) -> int32_t { + if (llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_POOLING_TYPE_UNKNOWN)) { + return LLAMA_POOLING_TYPE_RANK + 1; + } + + return llama_pooling_type(ctx); +} + +extern "C" auto llama_rs_model_rope_type(const struct llama_model * model) -> int32_t { + if (llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_ROPE_TYPE_UNKNOWN)) { + return LLAMA_ROPE_TYPE_IMROPE + 1; + } + + return llama_model_rope_type(model); +} + extern "C" auto llama_rs_context_produced_logits(struct llama_context * ctx) -> bool { return llama_get_logits(ctx) != nullptr; } diff --git a/llama-cpp-bindings-sys/wrapper_context.h b/llama-cpp-bindings-sys/wrapper_context.h index 7516e6bbf..0f944df77 100644 --- a/llama-cpp-bindings-sys/wrapper_context.h +++ b/llama-cpp-bindings-sys/wrapper_context.h @@ -14,6 +14,10 @@ struct llama_model; bool llama_rs_model_causal_attn(const struct llama_model * model); +int32_t llama_rs_pooling_type(const struct llama_context * ctx); + +int32_t llama_rs_model_rope_type(const struct llama_model * model); + bool llama_rs_context_produced_logits(struct llama_context * ctx); float * llama_rs_get_logits_ith(struct llama_context * ctx, int32_t token_index); diff --git a/llama-cpp-bindings-sys/wrapper_fault_injection.h b/llama-cpp-bindings-sys/wrapper_fault_injection.h index 71b144958..e8c07424b 100644 --- a/llama-cpp-bindings-sys/wrapper_fault_injection.h +++ b/llama-cpp-bindings-sys/wrapper_fault_injection.h @@ -9,6 +9,8 @@ extern "C" { typedef enum llama_rs_fault_point { LLAMA_RS_FAULT_POINT_NONE = 0, + LLAMA_RS_FAULT_POINT_BACKEND_DEVICE_TEXT_NULL, + LLAMA_RS_FAULT_POINT_BACKEND_DEVICE_TYPE_UNKNOWN, LLAMA_RS_FAULT_POINT_BATCH_EXT_ASSIGN_ENTRIES_THROWS, LLAMA_RS_FAULT_POINT_BATCH_EXT_INIT_THROWS, LLAMA_RS_FAULT_POINT_CHAT_SESSION_CLASSIFY_THROWS, @@ -26,11 +28,13 @@ typedef enum llama_rs_fault_point { LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNKS_INIT_THROWS, LLAMA_RS_FAULT_POINT_MTMD_TOKENIZE_FROM_PARTS_THROWS, LLAMA_RS_FAULT_POINT_PARSED_CHAT_CONTENT_NOT_UTF8, + LLAMA_RS_FAULT_POINT_POOLING_TYPE_UNKNOWN, LLAMA_RS_FAULT_POINT_PARSED_CHAT_REASONING_CONTENT_NOT_UTF8, LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NOT_UTF8, LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_ID_NOT_UTF8, LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_NAME_NOT_UTF8, LLAMA_RS_FAULT_POINT_REGEX_ESCAPE_THROWS, + LLAMA_RS_FAULT_POINT_ROPE_TYPE_UNKNOWN, LLAMA_RS_FAULT_POINT_SAMPLER_ACCEPT_THROWS, LLAMA_RS_FAULT_POINT_SAMPLER_APPLY_THROWS, LLAMA_RS_FAULT_POINT_SAMPLER_INIT_DIST_THROWS, diff --git a/llama-cpp-bindings-tests/tests/backend_device_listing.rs b/llama-cpp-bindings-tests/tests/backend_device_listing.rs new file mode 100644 index 000000000..c59667d45 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/backend_device_listing.rs @@ -0,0 +1,59 @@ +use llama_cpp_bindings::error::backend_device_text_error::BackendDeviceTextError; +use llama_cpp_bindings::error::llama_backend_device_error::LlamaBackendDeviceError; +use llama_cpp_bindings::error::unknown_backend_device_type::UnknownBackendDeviceType; +use llama_cpp_bindings::llama_backend_device::LlamaBackendDevice; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +fn listing_with_missing_text_after(passing_hits: usize) -> Option { + let _fault = ArmedFault::after_passing( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_BACKEND_DEVICE_TEXT_NULL, + passing_hits, + ); + + LlamaBackendDevice::list_all().err() +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn a_device_text_ggml_does_not_report_refuses_the_listing(_fixture: &LlamaFixture<'_>) { + assert_eq!( + [0, 1, 2].map(listing_with_missing_text_after), + [ + Some(LlamaBackendDeviceError::Name(BackendDeviceTextError::Null)), + Some(LlamaBackendDeviceError::Description( + BackendDeviceTextError::Null + )), + Some(LlamaBackendDeviceError::BackendName( + BackendDeviceTextError::Null + )), + ] + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn a_device_type_ggml_does_not_define_refuses_the_listing(_fixture: &LlamaFixture<'_>) { + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_BACKEND_DEVICE_TYPE_UNKNOWN); + + assert_eq!( + LlamaBackendDevice::list_all().err(), + Some(LlamaBackendDeviceError::UnknownDeviceType( + UnknownBackendDeviceType(llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_META + 1) + )) + ); +} diff --git a/llama-cpp-bindings-tests/tests/context_creation_limits.rs b/llama-cpp-bindings-tests/tests/context_creation_limits.rs index 1293ddd8b..9e4f0508d 100644 --- a/llama-cpp-bindings-tests/tests/context_creation_limits.rs +++ b/llama-cpp-bindings-tests/tests/context_creation_limits.rs @@ -2,6 +2,8 @@ use std::num::NonZeroU32; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::error::llama_context_load_error::LlamaContextLoadError; +use llama_cpp_bindings::error::unknown_attention_type::UnknownAttentionType; +use llama_cpp_bindings::error::unknown_pooling_type::UnknownPoolingType; use llama_cpp_fault_injection::armed_fault::ArmedFault; use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; use llama_cpp_test_harness::llama_fixture::LlamaFixture; @@ -122,3 +124,44 @@ fn creating_a_context_reports_a_batch_llama_cpp_cannot_allocate(fixture: &LlamaF }) ); } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn a_context_whose_pooling_type_llama_cpp_does_not_define_is_refused(fixture: &LlamaFixture<'_>) { + let _fault = ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_POOLING_TYPE_UNKNOWN); + + assert_eq!( + fixture.build_context().err(), + Some(LlamaContextLoadError::UnknownPoolingType( + UnknownPoolingType(llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_RANK + 1) + )) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn context_params_with_an_attention_type_llama_cpp_does_not_define_are_refused( + fixture: &LlamaFixture<'_>, +) { + let mut params = (*fixture.context_params).into_llama_context_params(); + params.context_params.attention_type = 99; + + assert_eq!( + LlamaContext::from_model(fixture.model, fixture.backend, params).err(), + Some(LlamaContextLoadError::UnknownAttentionType( + UnknownAttentionType(99) + )) + ); +} diff --git a/llama-cpp-bindings-tests/tests/main.rs b/llama-cpp-bindings-tests/tests/main.rs index ceaa6a1cd..8eb28ab1b 100644 --- a/llama-cpp-bindings-tests/tests/main.rs +++ b/llama-cpp-bindings-tests/tests/main.rs @@ -1,4 +1,5 @@ mod approximate_tok_env_eos_selection; +mod backend_device_listing; mod batch_context_limits; mod chat_answer_parsing; mod chat_prompt_rendering; diff --git a/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs b/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs index 3117e9f7a..1faa7e308 100644 --- a/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs +++ b/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs @@ -5,7 +5,9 @@ use anyhow::Result; use llama_cpp_bindings::error::memory_seq_add_error::MemorySeqAddError; use llama_cpp_bindings::error::memory_seq_div_error::MemorySeqDivError; +use llama_cpp_bindings::error::unknown_rope_type::UnknownRopeType; use llama_cpp_bindings_tests::prime_memory::prime_memory; +use llama_cpp_fault_injection::armed_fault::ArmedFault; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -117,3 +119,51 @@ fn memory_seq_div_divides_the_positions_of_a_standard_rope_model( Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_add_refuses_a_model_whose_rope_type_llama_cpp_does_not_define( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + let _fault = ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_ROPE_TYPE_UNKNOWN); + + assert_eq!( + context.memory_seq_add(0, None, None, 1), + Err(MemorySeqAddError::UnknownRopeType(UnknownRopeType( + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_IMROPE + 1 + ))) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_div_refuses_a_model_whose_rope_type_llama_cpp_does_not_define( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + let _fault = ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_ROPE_TYPE_UNKNOWN); + + assert_eq!( + context.memory_seq_div(0, None, None, NonZeroU8::MIN), + Err(MemorySeqDivError::UnknownRopeType(UnknownRopeType( + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_IMROPE + 1 + ))) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/model_architecture.rs b/llama-cpp-bindings-tests/tests/model_architecture.rs index 47a4b440a..775a9c662 100644 --- a/llama-cpp-bindings-tests/tests/model_architecture.rs +++ b/llama-cpp-bindings-tests/tests/model_architecture.rs @@ -92,6 +92,10 @@ fn is_hybrid_returns_true_for_qwen_default_models(fixture: &LlamaFixture<'_>) { n_batch = 128, n_ubatch = 64, )] +fn rope_type_reports_normal_rope_for_a_llama_architecture_model(fixture: &LlamaFixture<'_>) { + assert_eq!(fixture.model.rope_type(), Ok(RopeType::Norm)); +} + #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -100,15 +104,8 @@ fn is_hybrid_returns_true_for_qwen_default_models(fixture: &LlamaFixture<'_>) { n_batch = 128, n_ubatch = 64, )] -fn rope_type_returns_a_known_variant_for_rope_carrying_default_models(fixture: &LlamaFixture<'_>) { - let rope = fixture.model.rope_type(); - assert!( - matches!( - rope, - Some(RopeType::Norm | RopeType::NeoX | RopeType::MRope | RopeType::Vision) - ), - "rope_type must be a known variant for DeepSeek and GLM-4.7; got {rope:?}" - ); +fn rope_type_reports_normal_rope_for_glm47_flash(fixture: &LlamaFixture<'_>) { + assert_eq!(fixture.model.rope_type(), Ok(RopeType::Norm)); } #[llama_test( @@ -128,5 +125,5 @@ fn rope_type_returns_a_known_variant_for_rope_carrying_default_models(fixture: & n_ubatch = 64, )] fn rope_type_reports_interleaved_mrope_for_qwen_default_models(fixture: &LlamaFixture<'_>) { - assert_eq!(fixture.model.rope_type(), Some(RopeType::IMRope)); + assert_eq!(fixture.model.rope_type(), Ok(RopeType::IMRope)); } diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index 7cdf0f0d1..68cd792dc 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -297,6 +297,12 @@ impl<'model> LlamaContext<'model> { context: NonNull, embeddings: bool, ) -> Result { + let pooling_type = LlamaPoolingType::try_from(unsafe { + llama_cpp_bindings_sys::llama_rs_pooling_type(context.as_ptr()) + }) + .inspect_err(|_unknown_pooling_type| unsafe { + llama_cpp_bindings_sys::llama_free(context.as_ptr()); + })?; let mut out_batch: *mut llama_cpp_bindings_sys::llama_batch_ext = ptr::null_mut(); let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { @@ -311,9 +317,6 @@ impl<'model> LlamaContext<'model> { llama_cpp_bindings_sys::llama_free(context.as_ptr()); }, )?; - let pooling_type = LlamaPoolingType::from(unsafe { - llama_cpp_bindings_sys::llama_pooling_type(context.as_ptr()) - }); let keeps_no_memory = memory_of(context).is_none(); Ok(Self { @@ -349,7 +352,7 @@ impl<'model> LlamaContext<'model> { let embeddings = params.embeddings(); SequenceOutputCapacity { - attention_type: params.attention_type(), + attention_type: params.attention_type()?, model_causal_attention: unsafe { llama_cpp_bindings_sys::llama_rs_model_causal_attn(model.model.as_ptr()) }, diff --git a/llama-cpp-bindings/src/context/llama_attention_type.rs b/llama-cpp-bindings/src/context/llama_attention_type.rs index 3afc344fc..5a233573f 100644 --- a/llama-cpp-bindings/src/context/llama_attention_type.rs +++ b/llama-cpp-bindings/src/context/llama_attention_type.rs @@ -1,27 +1,35 @@ -#[repr(i8)] +use crate::error::unknown_attention_type::UnknownAttentionType; + #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum LlamaAttentionType { - Unspecified = -1, - Causal = 0, - NonCausal = 1, + Unspecified, + Causal, + NonCausal, } -impl From for LlamaAttentionType { - fn from(value: i32) -> Self { +impl TryFrom for LlamaAttentionType { + type Error = UnknownAttentionType; + + fn try_from(value: llama_cpp_bindings_sys::llama_attention_type) -> Result { match value { - 0 => Self::Causal, - 1 => Self::NonCausal, - _ => Self::Unspecified, + llama_cpp_bindings_sys::LLAMA_ATTENTION_TYPE_UNSPECIFIED => Ok(Self::Unspecified), + llama_cpp_bindings_sys::LLAMA_ATTENTION_TYPE_CAUSAL => Ok(Self::Causal), + llama_cpp_bindings_sys::LLAMA_ATTENTION_TYPE_NON_CAUSAL => Ok(Self::NonCausal), + unknown => Err(UnknownAttentionType(unknown)), } } } -impl From for i32 { +impl From for llama_cpp_bindings_sys::llama_attention_type { fn from(value: LlamaAttentionType) -> Self { match value { - LlamaAttentionType::Causal => 0, - LlamaAttentionType::NonCausal => 1, - LlamaAttentionType::Unspecified => -1, + LlamaAttentionType::Unspecified => { + llama_cpp_bindings_sys::LLAMA_ATTENTION_TYPE_UNSPECIFIED + } + LlamaAttentionType::Causal => llama_cpp_bindings_sys::LLAMA_ATTENTION_TYPE_CAUSAL, + LlamaAttentionType::NonCausal => { + llama_cpp_bindings_sys::LLAMA_ATTENTION_TYPE_NON_CAUSAL + } } } } @@ -29,31 +37,40 @@ impl From for i32 { #[cfg(test)] mod tests { use super::LlamaAttentionType; + use crate::error::unknown_attention_type::UnknownAttentionType; #[test] - fn attention_type_unknown_defaults_to_unspecified() { + fn every_llama_cpp_value_converts_to_its_variant_and_back() { + let raw_values = [ + llama_cpp_bindings_sys::LLAMA_ATTENTION_TYPE_UNSPECIFIED, + llama_cpp_bindings_sys::LLAMA_ATTENTION_TYPE_CAUSAL, + llama_cpp_bindings_sys::LLAMA_ATTENTION_TYPE_NON_CAUSAL, + ]; + assert_eq!( - LlamaAttentionType::from(99), - LlamaAttentionType::Unspecified + raw_values.map(LlamaAttentionType::try_from), + [ + Ok(LlamaAttentionType::Unspecified), + Ok(LlamaAttentionType::Causal), + Ok(LlamaAttentionType::NonCausal), + ] ); assert_eq!( - LlamaAttentionType::from(-50), - LlamaAttentionType::Unspecified + [ + LlamaAttentionType::Unspecified, + LlamaAttentionType::Causal, + LlamaAttentionType::NonCausal, + ] + .map(llama_cpp_bindings_sys::llama_attention_type::from), + raw_values ); } #[test] - fn every_attention_type_converts_to_llama_cpp_and_back() { - for (raw, expected) in [ - (-1, LlamaAttentionType::Unspecified), - (0, LlamaAttentionType::Causal), - (1, LlamaAttentionType::NonCausal), - ] { - let from_raw = LlamaAttentionType::from(raw); - assert_eq!(from_raw, expected); - - let back_to_raw: i32 = from_raw.into(); - assert_eq!(back_to_raw, raw); - } + fn a_value_llama_cpp_does_not_define_is_refused() { + assert_eq!( + LlamaAttentionType::try_from(99), + Err(UnknownAttentionType(99)) + ); } } diff --git a/llama-cpp-bindings/src/context/llama_pooling_type.rs b/llama-cpp-bindings/src/context/llama_pooling_type.rs index 38f9c423e..ce1983f3c 100644 --- a/llama-cpp-bindings/src/context/llama_pooling_type.rs +++ b/llama-cpp-bindings/src/context/llama_pooling_type.rs @@ -1,36 +1,40 @@ -#[repr(i8)] +use crate::error::unknown_pooling_type::UnknownPoolingType; + #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum LlamaPoolingType { - Unspecified = -1, - None = 0, - Mean = 1, - Cls = 2, - Last = 3, - Rank = 4, + Unspecified, + None, + Mean, + Cls, + Last, + Rank, } -impl From for LlamaPoolingType { - fn from(value: i32) -> Self { +impl TryFrom for LlamaPoolingType { + type Error = UnknownPoolingType; + + fn try_from(value: llama_cpp_bindings_sys::llama_pooling_type) -> Result { match value { - 0 => Self::None, - 1 => Self::Mean, - 2 => Self::Cls, - 3 => Self::Last, - 4 => Self::Rank, - _ => Self::Unspecified, + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_UNSPECIFIED => Ok(Self::Unspecified), + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_NONE => Ok(Self::None), + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_MEAN => Ok(Self::Mean), + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_CLS => Ok(Self::Cls), + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_LAST => Ok(Self::Last), + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_RANK => Ok(Self::Rank), + unknown => Err(UnknownPoolingType(unknown)), } } } -impl From for i32 { +impl From for llama_cpp_bindings_sys::llama_pooling_type { fn from(value: LlamaPoolingType) -> Self { match value { - LlamaPoolingType::None => 0, - LlamaPoolingType::Mean => 1, - LlamaPoolingType::Cls => 2, - LlamaPoolingType::Last => 3, - LlamaPoolingType::Rank => 4, - LlamaPoolingType::Unspecified => -1, + LlamaPoolingType::Unspecified => llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_UNSPECIFIED, + LlamaPoolingType::None => llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_NONE, + LlamaPoolingType::Mean => llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_MEAN, + LlamaPoolingType::Cls => llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_CLS, + LlamaPoolingType::Last => llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_LAST, + LlamaPoolingType::Rank => llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_RANK, } } } @@ -38,28 +42,46 @@ impl From for i32 { #[cfg(test)] mod tests { use super::LlamaPoolingType; + use crate::error::unknown_pooling_type::UnknownPoolingType; #[test] - fn pooling_type_unknown_defaults_to_unspecified() { - assert_eq!(LlamaPoolingType::from(99), LlamaPoolingType::Unspecified); - assert_eq!(LlamaPoolingType::from(-50), LlamaPoolingType::Unspecified); + fn every_llama_cpp_value_converts_to_its_variant_and_back() { + let raw_values = [ + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_UNSPECIFIED, + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_NONE, + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_MEAN, + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_CLS, + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_LAST, + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_RANK, + ]; + + assert_eq!( + raw_values.map(LlamaPoolingType::try_from), + [ + Ok(LlamaPoolingType::Unspecified), + Ok(LlamaPoolingType::None), + Ok(LlamaPoolingType::Mean), + Ok(LlamaPoolingType::Cls), + Ok(LlamaPoolingType::Last), + Ok(LlamaPoolingType::Rank), + ] + ); + assert_eq!( + [ + LlamaPoolingType::Unspecified, + LlamaPoolingType::None, + LlamaPoolingType::Mean, + LlamaPoolingType::Cls, + LlamaPoolingType::Last, + LlamaPoolingType::Rank, + ] + .map(llama_cpp_bindings_sys::llama_pooling_type::from), + raw_values + ); } #[test] - fn every_pooling_type_converts_to_llama_cpp_and_back() { - for (raw, expected) in [ - (-1, LlamaPoolingType::Unspecified), - (0, LlamaPoolingType::None), - (1, LlamaPoolingType::Mean), - (2, LlamaPoolingType::Cls), - (3, LlamaPoolingType::Last), - (4, LlamaPoolingType::Rank), - ] { - let from_raw = LlamaPoolingType::from(raw); - assert_eq!(from_raw, expected); - - let back_to_raw: i32 = from_raw.into(); - assert_eq!(back_to_raw, raw); - } + fn a_value_llama_cpp_does_not_define_is_refused() { + assert_eq!(LlamaPoolingType::try_from(99), Err(UnknownPoolingType(99))); } } diff --git a/llama-cpp-bindings/src/context/memory.rs b/llama-cpp-bindings/src/context/memory.rs index c650edc86..6ee0f2d89 100644 --- a/llama-cpp-bindings/src/context/memory.rs +++ b/llama-cpp-bindings/src/context/memory.rs @@ -15,6 +15,7 @@ use crate::error::memory_seq_div_error::MemorySeqDivError; use crate::error::memory_seq_pos_max_error::MemorySeqPosMaxError; use crate::error::memory_seq_rm_error::MemorySeqRmError; use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; +use crate::error::unknown_rope_type::UnknownRopeType; use crate::model::rope_type::RopeType; fn memory_seq_add_status_to_result( @@ -152,10 +153,10 @@ impl LlamaContext<'_> { memory_of(self.context).is_some() } - fn shifts_positions_one_per_embedding(&self) -> bool { + fn shifts_positions_one_per_embedding(&self) -> Result { self.model .rope_type() - .is_none_or(RopeType::has_one_position_per_embedding) + .map(RopeType::has_one_position_per_embedding) } fn required_memory_handle( @@ -264,7 +265,7 @@ impl LlamaContext<'_> { .map_or(Ok(-1), i32::try_from) .map_err(MemorySeqAddError::P1TooLarge)?; - if !self.shifts_positions_one_per_embedding() { + if !self.shifts_positions_one_per_embedding()? { return Err(MemorySeqAddError::IncompatibleRopeType); } @@ -300,7 +301,7 @@ impl LlamaContext<'_> { .map_or(Ok(-1), i32::try_from) .map_err(MemorySeqDivError::P1TooLarge)?; - if !self.shifts_positions_one_per_embedding() { + if !self.shifts_positions_one_per_embedding()? { return Err(MemorySeqDivError::IncompatibleRopeType); } diff --git a/llama-cpp-bindings/src/context/params.rs b/llama-cpp-bindings/src/context/params.rs index 2d8495c4e..462c1beaa 100644 --- a/llama-cpp-bindings/src/context/params.rs +++ b/llama-cpp-bindings/src/context/params.rs @@ -6,6 +6,9 @@ use crate::context::kv_cache_type::KvCacheType; use crate::context::llama_attention_type::LlamaAttentionType; use crate::context::llama_pooling_type::LlamaPoolingType; use crate::context::rope_scaling_type::RopeScalingType; +use crate::error::unknown_attention_type::UnknownAttentionType; +use crate::error::unknown_pooling_type::UnknownPoolingType; +use crate::error::unknown_rope_scaling_type::UnknownRopeScalingType; #[derive(Debug, Clone, Copy)] pub struct LlamaContextParams { @@ -80,9 +83,12 @@ impl LlamaContextParams { self } - #[must_use] - pub fn rope_scaling_type(&self) -> RopeScalingType { - RopeScalingType::from(self.context_params.rope_scaling_type) + /// # Errors + /// + /// Returns [`UnknownRopeScalingType`] when the raw params hold a value llama.cpp does not + /// define. + pub fn rope_scaling_type(&self) -> Result { + RopeScalingType::try_from(self.context_params.rope_scaling_type) } #[must_use] @@ -161,9 +167,11 @@ impl LlamaContextParams { self } - #[must_use] - pub fn pooling_type(&self) -> LlamaPoolingType { - LlamaPoolingType::from(self.context_params.pooling_type) + /// # Errors + /// + /// Returns [`UnknownPoolingType`] when the raw params hold a value llama.cpp does not define. + pub fn pooling_type(&self) -> Result { + LlamaPoolingType::try_from(self.context_params.pooling_type) } #[must_use] @@ -228,9 +236,12 @@ impl LlamaContextParams { self } - #[must_use] - pub fn attention_type(&self) -> LlamaAttentionType { - LlamaAttentionType::from(self.context_params.attention_type) + /// # Errors + /// + /// Returns [`UnknownAttentionType`] when the raw params hold a value llama.cpp does not + /// define. + pub fn attention_type(&self) -> Result { + LlamaAttentionType::try_from(self.context_params.attention_type) } #[must_use] @@ -336,8 +347,8 @@ mod tests { assert_eq!(params.n_ctx(), NonZeroU32::new(512)); assert_eq!(params.n_batch(), 2048); assert_eq!(params.n_ubatch(), 512); - assert_eq!(params.rope_scaling_type(), RopeScalingType::Unspecified); - assert_eq!(params.pooling_type(), LlamaPoolingType::Unspecified); + assert_eq!(params.rope_scaling_type(), Ok(RopeScalingType::Unspecified)); + assert_eq!(params.pooling_type(), Ok(LlamaPoolingType::Unspecified)); } #[test] @@ -437,7 +448,7 @@ mod tests { .with_rope_scaling_type(scaling_type) .rope_scaling_type() }), - scaling_types + scaling_types.map(Ok) ); } @@ -485,7 +496,7 @@ mod tests { .with_pooling_type(pooling_type) .pooling_type() }), - pooling_types + pooling_types.map(Ok) ); } @@ -548,7 +559,7 @@ mod tests { assert_eq!(params.n_threads_batch(), 12); assert!(params.embeddings()); assert!(!params.offload_kqv()); - assert_eq!(params.rope_scaling_type(), RopeScalingType::Yarn); + assert_eq!(params.rope_scaling_type(), Ok(RopeScalingType::Yarn)); assert!((params.rope_freq_base() - 5000.0).abs() < f32::EPSILON); assert!((params.rope_freq_scale() - 0.25).abs() < f32::EPSILON); } @@ -593,7 +604,7 @@ mod tests { .with_attention_type(attention_type) .attention_type() }), - attention_types + attention_types.map(Ok) ); } diff --git a/llama-cpp-bindings/src/context/rope_scaling_type.rs b/llama-cpp-bindings/src/context/rope_scaling_type.rs index 8a0ea0577..99edbcf38 100644 --- a/llama-cpp-bindings/src/context/rope_scaling_type.rs +++ b/llama-cpp-bindings/src/context/rope_scaling_type.rs @@ -1,30 +1,41 @@ -#[repr(i8)] +use crate::error::unknown_rope_scaling_type::UnknownRopeScalingType; + #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum RopeScalingType { - Unspecified = -1, - None = 0, - Linear = 1, - Yarn = 2, + Unspecified, + None, + Linear, + Yarn, + LongRope, } -impl From for RopeScalingType { - fn from(value: i32) -> Self { +impl TryFrom for RopeScalingType { + type Error = UnknownRopeScalingType; + + fn try_from( + value: llama_cpp_bindings_sys::llama_rope_scaling_type, + ) -> Result { match value { - 0 => Self::None, - 1 => Self::Linear, - 2 => Self::Yarn, - _ => Self::Unspecified, + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED => Ok(Self::Unspecified), + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_NONE => Ok(Self::None), + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_LINEAR => Ok(Self::Linear), + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_YARN => Ok(Self::Yarn), + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_LONGROPE => Ok(Self::LongRope), + unknown => Err(UnknownRopeScalingType(unknown)), } } } -impl From for i32 { +impl From for llama_cpp_bindings_sys::llama_rope_scaling_type { fn from(value: RopeScalingType) -> Self { match value { - RopeScalingType::None => 0, - RopeScalingType::Linear => 1, - RopeScalingType::Yarn => 2, - RopeScalingType::Unspecified => -1, + RopeScalingType::Unspecified => { + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED + } + RopeScalingType::None => llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_NONE, + RopeScalingType::Linear => llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_LINEAR, + RopeScalingType::Yarn => llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_YARN, + RopeScalingType::LongRope => llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_LONGROPE, } } } @@ -32,26 +43,46 @@ impl From for i32 { #[cfg(test)] mod tests { use super::RopeScalingType; + use crate::error::unknown_rope_scaling_type::UnknownRopeScalingType; #[test] - fn rope_scaling_type_unknown_defaults_to_unspecified() { - assert_eq!(RopeScalingType::from(99), RopeScalingType::Unspecified); - assert_eq!(RopeScalingType::from(-100), RopeScalingType::Unspecified); + fn every_llama_cpp_value_converts_to_its_variant_and_back() { + let raw_values = [ + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_NONE, + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_LINEAR, + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_YARN, + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_LONGROPE, + ]; + + assert_eq!( + raw_values.map(RopeScalingType::try_from), + [ + Ok(RopeScalingType::Unspecified), + Ok(RopeScalingType::None), + Ok(RopeScalingType::Linear), + Ok(RopeScalingType::Yarn), + Ok(RopeScalingType::LongRope), + ] + ); + assert_eq!( + [ + RopeScalingType::Unspecified, + RopeScalingType::None, + RopeScalingType::Linear, + RopeScalingType::Yarn, + RopeScalingType::LongRope, + ] + .map(llama_cpp_bindings_sys::llama_rope_scaling_type::from), + raw_values + ); } #[test] - fn every_rope_scaling_type_converts_to_llama_cpp_and_back() { - for (raw, expected) in [ - (-1, RopeScalingType::Unspecified), - (0, RopeScalingType::None), - (1, RopeScalingType::Linear), - (2, RopeScalingType::Yarn), - ] { - let from_raw = RopeScalingType::from(raw); - assert_eq!(from_raw, expected); - - let back_to_raw: i32 = from_raw.into(); - assert_eq!(back_to_raw, raw); - } + fn a_value_llama_cpp_does_not_define_is_refused() { + assert_eq!( + RopeScalingType::try_from(99), + Err(UnknownRopeScalingType(99)) + ); } } diff --git a/llama-cpp-bindings/src/error.rs b/llama-cpp-bindings/src/error.rs index 7e6666cdc..9260a422f 100644 --- a/llama-cpp-bindings/src/error.rs +++ b/llama-cpp-bindings/src/error.rs @@ -1,3 +1,4 @@ +pub mod backend_device_text_error; pub mod batch_transfer_error; pub mod chat_session_error; pub mod chat_template_error; @@ -11,6 +12,7 @@ pub mod grammar_error; pub mod grammar_runtime_error; pub mod ingest_error; pub mod json_schema_to_grammar_error; +pub mod llama_backend_device_error; pub mod llama_context_load_error; pub mod llama_cpp_error; pub mod llama_lora_adapter_init_error; @@ -36,6 +38,11 @@ pub mod string_to_token_error; pub mod token_attr_error; pub mod token_outside_vocabulary; pub mod token_to_string_error; +pub mod unknown_attention_type; +pub mod unknown_backend_device_type; +pub mod unknown_pooling_type; +pub mod unknown_rope_scaling_type; +pub mod unknown_rope_type; use llama_cpp_error::LlamaCppError; diff --git a/llama-cpp-bindings/src/error/backend_device_text_error.rs b/llama-cpp-bindings/src/error/backend_device_text_error.rs new file mode 100644 index 000000000..d9d5e9741 --- /dev/null +++ b/llama-cpp-bindings/src/error/backend_device_text_error.rs @@ -0,0 +1,9 @@ +use std::str::Utf8Error; + +#[derive(Debug, Eq, PartialEq, thiserror::Error)] +pub enum BackendDeviceTextError { + #[error("ggml reported no text")] + Null, + #[error("ggml reported text that is not valid UTF-8")] + NotUtf8(#[from] Utf8Error), +} diff --git a/llama-cpp-bindings/src/error/llama_backend_device_error.rs b/llama-cpp-bindings/src/error/llama_backend_device_error.rs new file mode 100644 index 000000000..25f9c550f --- /dev/null +++ b/llama-cpp-bindings/src/error/llama_backend_device_error.rs @@ -0,0 +1,14 @@ +use crate::error::backend_device_text_error::BackendDeviceTextError; +use crate::error::unknown_backend_device_type::UnknownBackendDeviceType; + +#[derive(Debug, Eq, PartialEq, thiserror::Error)] +pub enum LlamaBackendDeviceError { + #[error("the device name cannot be read: {0}")] + Name(#[source] BackendDeviceTextError), + #[error("the device description cannot be read: {0}")] + Description(#[source] BackendDeviceTextError), + #[error("the device's backend name cannot be read: {0}")] + BackendName(#[source] BackendDeviceTextError), + #[error(transparent)] + UnknownDeviceType(#[from] UnknownBackendDeviceType), +} diff --git a/llama-cpp-bindings/src/error/llama_context_load_error.rs b/llama-cpp-bindings/src/error/llama_context_load_error.rs index a112b096a..8fed19da9 100644 --- a/llama-cpp-bindings/src/error/llama_context_load_error.rs +++ b/llama-cpp-bindings/src/error/llama_context_load_error.rs @@ -1,6 +1,9 @@ use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use crate::error::unknown_attention_type::UnknownAttentionType; +use crate::error::unknown_pooling_type::UnknownPoolingType; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum LlamaContextLoadError { #[error(transparent)] @@ -22,4 +25,8 @@ pub enum LlamaContextLoadError { n_seq_max: u32, output_capacity: u32, }, + #[error(transparent)] + UnknownAttentionType(#[from] UnknownAttentionType), + #[error(transparent)] + UnknownPoolingType(#[from] UnknownPoolingType), } diff --git a/llama-cpp-bindings/src/error/memory_seq_add_error.rs b/llama-cpp-bindings/src/error/memory_seq_add_error.rs index b93298834..d81b4bfdb 100644 --- a/llama-cpp-bindings/src/error/memory_seq_add_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_add_error.rs @@ -4,6 +4,7 @@ use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; +use crate::error::unknown_rope_type::UnknownRopeType; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum MemorySeqAddError { @@ -21,6 +22,8 @@ pub enum MemorySeqAddError { MemoryHandleUnavailable, #[error(transparent)] SequenceIdOutOfRange(#[from] SequenceIdOutOfRange), + #[error(transparent)] + UnknownRopeType(#[from] UnknownRopeType), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/error/memory_seq_div_error.rs b/llama-cpp-bindings/src/error/memory_seq_div_error.rs index d428d6859..9a80bcd43 100644 --- a/llama-cpp-bindings/src/error/memory_seq_div_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_div_error.rs @@ -4,6 +4,7 @@ use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; +use crate::error::unknown_rope_type::UnknownRopeType; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum MemorySeqDivError { @@ -21,6 +22,8 @@ pub enum MemorySeqDivError { MemoryHandleUnavailable, #[error(transparent)] SequenceIdOutOfRange(#[from] SequenceIdOutOfRange), + #[error(transparent)] + UnknownRopeType(#[from] UnknownRopeType), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/error/unknown_attention_type.rs b/llama-cpp-bindings/src/error/unknown_attention_type.rs new file mode 100644 index 000000000..d48f1858d --- /dev/null +++ b/llama-cpp-bindings/src/error/unknown_attention_type.rs @@ -0,0 +1,3 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +#[error("attention type {0} is not a value llama.cpp defines")] +pub struct UnknownAttentionType(pub llama_cpp_bindings_sys::llama_attention_type); diff --git a/llama-cpp-bindings/src/error/unknown_backend_device_type.rs b/llama-cpp-bindings/src/error/unknown_backend_device_type.rs new file mode 100644 index 000000000..35fcca137 --- /dev/null +++ b/llama-cpp-bindings/src/error/unknown_backend_device_type.rs @@ -0,0 +1,3 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +#[error("backend device type {0} is not a value ggml defines")] +pub struct UnknownBackendDeviceType(pub llama_cpp_bindings_sys::ggml_backend_dev_type); diff --git a/llama-cpp-bindings/src/error/unknown_pooling_type.rs b/llama-cpp-bindings/src/error/unknown_pooling_type.rs new file mode 100644 index 000000000..69c18752c --- /dev/null +++ b/llama-cpp-bindings/src/error/unknown_pooling_type.rs @@ -0,0 +1,3 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +#[error("pooling type {0} is not a value llama.cpp defines")] +pub struct UnknownPoolingType(pub llama_cpp_bindings_sys::llama_pooling_type); diff --git a/llama-cpp-bindings/src/error/unknown_rope_scaling_type.rs b/llama-cpp-bindings/src/error/unknown_rope_scaling_type.rs new file mode 100644 index 000000000..fb523f16e --- /dev/null +++ b/llama-cpp-bindings/src/error/unknown_rope_scaling_type.rs @@ -0,0 +1,3 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +#[error("rope scaling type {0} is not a value llama.cpp defines")] +pub struct UnknownRopeScalingType(pub llama_cpp_bindings_sys::llama_rope_scaling_type); diff --git a/llama-cpp-bindings/src/error/unknown_rope_type.rs b/llama-cpp-bindings/src/error/unknown_rope_type.rs new file mode 100644 index 000000000..b09934d44 --- /dev/null +++ b/llama-cpp-bindings/src/error/unknown_rope_type.rs @@ -0,0 +1,3 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +#[error("rope type {0} is not a value llama.cpp defines")] +pub struct UnknownRopeType(pub llama_cpp_bindings_sys::llama_rope_type); diff --git a/llama-cpp-bindings/src/llama_backend.rs b/llama-cpp-bindings/src/llama_backend.rs index 06570afac..cbbc08aab 100644 --- a/llama-cpp-bindings/src/llama_backend.rs +++ b/llama-cpp-bindings/src/llama_backend.rs @@ -123,10 +123,13 @@ mod tests { assert_eq!( offloads_to_a_gpu, - LlamaBackendDevice::list_all().iter().any(|device| matches!( - device.device_type, - LlamaBackendDeviceType::Gpu | LlamaBackendDeviceType::IntegratedGpu - )) + LlamaBackendDevice::list_all() + .unwrap() + .iter() + .any(|device| matches!( + device.device_type, + LlamaBackendDeviceType::Gpu | LlamaBackendDeviceType::IntegratedGpu + )) ); } diff --git a/llama-cpp-bindings/src/llama_backend_device.rs b/llama-cpp-bindings/src/llama_backend_device.rs index 700fc0218..b91465451 100644 --- a/llama-cpp-bindings/src/llama_backend_device.rs +++ b/llama-cpp-bindings/src/llama_backend_device.rs @@ -2,14 +2,20 @@ use std::ffi::CStr; use std::ffi::c_char; use std::mem; +use crate::error::backend_device_text_error::BackendDeviceTextError; +use crate::error::llama_backend_device_error::LlamaBackendDeviceError; use crate::llama_backend_device_type::LlamaBackendDeviceType; -fn cstr_to_string(ptr: *const c_char) -> String { - if ptr.is_null() { - String::new() - } else { - unsafe { CStr::from_ptr(ptr) }.to_string_lossy().to_string() +fn text_of(text: *const c_char) -> Result { + if text.is_null() { + return Err(BackendDeviceTextError::Null); } + + Ok(unsafe { CStr::from_ptr(text) }.to_str()?.to_owned()) +} + +fn device_text(text: *const c_char) -> Result { + text_of(unsafe { llama_cpp_bindings_sys::llama_rs_backend_device_text(text) }) } #[derive(Debug, Clone)] @@ -24,39 +30,42 @@ pub struct LlamaBackendDevice { } impl LlamaBackendDevice { - #[must_use] - pub fn list_all() -> Vec { - let mut devices = Vec::new(); - let device_count = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_count() }; - - for device_index in 0..device_count { - let dev = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_get(device_index) }; - let props = unsafe { - let mut props = mem::zeroed(); - llama_cpp_bindings_sys::ggml_backend_dev_get_props(dev, &raw mut props); - props - }; - let name = cstr_to_string(props.name); - let description = cstr_to_string(props.description); - let backend_reg = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_backend_reg(dev) }; - let backend_name = - unsafe { llama_cpp_bindings_sys::ggml_backend_reg_name(backend_reg) }; - let backend = cstr_to_string(backend_name); - let memory_total = props.memory_total; - let memory_free = props.memory_free; - let device_type = LlamaBackendDeviceType::from_raw(props.type_); - devices.push(Self { - index: device_index, - name, - description, - backend, - memory_total, - memory_free, - device_type, - }); - } - - devices + fn at_index(device_index: usize) -> Result { + let dev = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_get(device_index) }; + let props = unsafe { + let mut props = mem::zeroed(); + llama_cpp_bindings_sys::ggml_backend_dev_get_props(dev, &raw mut props); + props + }; + let backend_reg = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_backend_reg(dev) }; + + Ok(Self { + index: device_index, + name: device_text(props.name).map_err(LlamaBackendDeviceError::Name)?, + description: device_text(props.description) + .map_err(LlamaBackendDeviceError::Description)?, + backend: device_text(unsafe { + llama_cpp_bindings_sys::ggml_backend_reg_name(backend_reg) + }) + .map_err(LlamaBackendDeviceError::BackendName)?, + memory_total: props.memory_total, + memory_free: props.memory_free, + device_type: LlamaBackendDeviceType::try_from(unsafe { + llama_cpp_bindings_sys::llama_rs_backend_device_type(dev) + })?, + }) + } + + /// Every device ggml registered, in registration order. + /// + /// # Errors + /// + /// Returns [`LlamaBackendDeviceError`] when ggml reports a device text that is missing or not + /// UTF-8, or a device type it does not define. + pub fn list_all() -> Result, LlamaBackendDeviceError> { + (0..unsafe { llama_cpp_bindings_sys::ggml_backend_dev_count() }) + .map(Self::at_index) + .collect() } } @@ -64,33 +73,42 @@ impl LlamaBackendDevice { mod tests { use std::ptr; - use super::cstr_to_string; + use super::text_of; + use crate::error::backend_device_text_error::BackendDeviceTextError; use crate::llama_backend_device::LlamaBackendDevice; - #[cfg(feature = "dynamic-backends")] - use crate::load_backends::load_backends; #[test] - fn cstr_to_string_with_null_returns_empty() { - let result = cstr_to_string(ptr::null()); - - assert_eq!(result, ""); + fn missing_device_text_is_refused() { + assert_eq!(text_of(ptr::null()), Err(BackendDeviceTextError::Null)); } #[test] - fn cstr_to_string_copies_a_non_null_string() { - let result = cstr_to_string(c"hello".as_ptr()); + fn device_text_that_is_not_utf8_is_refused() { + let not_utf8 = c"\xff"; - assert_eq!(result, "hello"); + assert_eq!( + text_of(not_utf8.as_ptr()), + Err(BackendDeviceTextError::NotUtf8( + not_utf8.to_str().expect_err("0xff is not UTF-8") + )) + ); } #[test] - fn list_all_lists_a_named_device_at_index_zero() { - #[cfg(feature = "dynamic-backends")] - load_backends().unwrap(); - - let devices = LlamaBackendDevice::list_all(); - assert!(!devices.is_empty()); - assert_eq!(devices[0].index, 0); - assert_ne!(devices[0].name, ""); + fn device_text_is_copied() { + assert_eq!(text_of(c"CUDA0".as_ptr()), Ok("CUDA0".to_owned())); + } + + #[test] + fn every_registered_device_is_listed_at_its_index() { + let devices = LlamaBackendDevice::list_all().unwrap(); + + assert_eq!( + devices + .iter() + .map(|device| device.index) + .collect::>(), + (0..devices.len()).collect::>() + ); } } diff --git a/llama-cpp-bindings/src/llama_backend_device_type.rs b/llama-cpp-bindings/src/llama_backend_device_type.rs index 82fefca7a..779828d8e 100644 --- a/llama-cpp-bindings/src/llama_backend_device_type.rs +++ b/llama-cpp-bindings/src/llama_backend_device_type.rs @@ -1,52 +1,60 @@ +use crate::error::unknown_backend_device_type::UnknownBackendDeviceType; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum LlamaBackendDeviceType { Cpu, - Accelerator, Gpu, IntegratedGpu, - Unknown, + Accelerator, + Meta, } -impl LlamaBackendDeviceType { - #[must_use] - pub const fn from_raw(raw_type: llama_cpp_bindings_sys::ggml_backend_dev_type) -> Self { - match raw_type { - llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_CPU => Self::Cpu, - llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_ACCEL => Self::Accelerator, - llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_GPU => Self::Gpu, - llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_IGPU => Self::IntegratedGpu, - _ => Self::Unknown, +impl TryFrom for LlamaBackendDeviceType { + type Error = UnknownBackendDeviceType; + + fn try_from(value: llama_cpp_bindings_sys::ggml_backend_dev_type) -> Result { + match value { + llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_CPU => Ok(Self::Cpu), + llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_GPU => Ok(Self::Gpu), + llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_IGPU => Ok(Self::IntegratedGpu), + llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_ACCEL => Ok(Self::Accelerator), + llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_META => Ok(Self::Meta), + unknown => Err(UnknownBackendDeviceType(unknown)), } } } #[cfg(test)] mod tests { + use crate::error::unknown_backend_device_type::UnknownBackendDeviceType; use crate::llama_backend_device_type::LlamaBackendDeviceType; #[test] - fn from_raw_maps_every_device_type_and_an_unknown_one() { + fn every_ggml_device_type_converts_to_its_variant() { assert_eq!( - LlamaBackendDeviceType::from_raw(llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_CPU), - LlamaBackendDeviceType::Cpu - ); - assert_eq!( - LlamaBackendDeviceType::from_raw( - llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_ACCEL - ), - LlamaBackendDeviceType::Accelerator - ); - assert_eq!( - LlamaBackendDeviceType::from_raw(llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_GPU), - LlamaBackendDeviceType::Gpu - ); - assert_eq!( - LlamaBackendDeviceType::from_raw(llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_IGPU), - LlamaBackendDeviceType::IntegratedGpu + [ + llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_CPU, + llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_GPU, + llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_IGPU, + llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_ACCEL, + llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_META, + ] + .map(LlamaBackendDeviceType::try_from), + [ + Ok(LlamaBackendDeviceType::Cpu), + Ok(LlamaBackendDeviceType::Gpu), + Ok(LlamaBackendDeviceType::IntegratedGpu), + Ok(LlamaBackendDeviceType::Accelerator), + Ok(LlamaBackendDeviceType::Meta), + ] ); + } + + #[test] + fn a_value_ggml_does_not_define_is_refused() { assert_eq!( - LlamaBackendDeviceType::from_raw(9999), - LlamaBackendDeviceType::Unknown + LlamaBackendDeviceType::try_from(9999), + Err(UnknownBackendDeviceType(9999)) ); } } diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index 6388627d4..f52bed550 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -51,6 +51,7 @@ use crate::error::string_to_token_error::StringToTokenError; use crate::error::token_attr_error::TokenAttrError; use crate::error::token_outside_vocabulary::TokenOutsideVocabulary; use crate::error::token_to_string_error::TokenToStringError; +use crate::error::unknown_rope_type::UnknownRopeType; use crate::llama_backend::LlamaBackend; use crate::llama_token_attrs::LlamaTokenAttrs; use crate::model::add_bos::AddBos; @@ -571,11 +572,13 @@ impl LlamaModel { ) } - #[must_use] - pub fn rope_type(&self) -> Option { - let raw = unsafe { llama_cpp_bindings_sys::llama_model_rope_type(self.model.as_ptr()) }; - - RopeType::from_raw(raw) + /// # Errors + /// + /// Returns [`UnknownRopeType`] when llama.cpp reports a rope type it does not define. + pub fn rope_type(&self) -> Result { + RopeType::try_from(unsafe { + llama_cpp_bindings_sys::llama_rs_model_rope_type(self.model.as_ptr()) + }) } /// # Errors diff --git a/llama-cpp-bindings/src/model/rope_type.rs b/llama-cpp-bindings/src/model/rope_type.rs index 53b7af021..b3d75de40 100644 --- a/llama-cpp-bindings/src/model/rope_type.rs +++ b/llama-cpp-bindings/src/model/rope_type.rs @@ -1,5 +1,8 @@ +use crate::error::unknown_rope_type::UnknownRopeType; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RopeType { + None, Norm, NeoX, MRope, @@ -7,25 +10,29 @@ pub enum RopeType { Vision, } -impl RopeType { - #[must_use] - pub const fn from_raw(raw: i32) -> Option { - match raw { - llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NORM => Some(Self::Norm), - llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NEOX => Some(Self::NeoX), - llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_MROPE => Some(Self::MRope), - llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_IMROPE => Some(Self::IMRope), - llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_VISION => Some(Self::Vision), - _ => None, +impl TryFrom for RopeType { + type Error = UnknownRopeType; + + fn try_from(value: llama_cpp_bindings_sys::llama_rope_type) -> Result { + match value { + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NONE => Ok(Self::None), + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NORM => Ok(Self::Norm), + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NEOX => Ok(Self::NeoX), + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_MROPE => Ok(Self::MRope), + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_IMROPE => Ok(Self::IMRope), + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_VISION => Ok(Self::Vision), + unknown => Err(UnknownRopeType(unknown)), } } +} +impl RopeType { /// Whether every embedding carries a single position, which llama.cpp requires before it /// shifts or divides the positions of a sequence in memory. #[must_use] pub const fn has_one_position_per_embedding(self) -> bool { match self { - Self::Norm | Self::NeoX | Self::Vision => true, + Self::None | Self::Norm | Self::NeoX | Self::Vision => true, Self::MRope | Self::IMRope => false, } } @@ -33,37 +40,42 @@ impl RopeType { #[cfg(test)] mod tests { + use crate::error::unknown_rope_type::UnknownRopeType; use crate::model::rope_type::RopeType; #[test] - fn llama_cpp_rope_types_map_to_their_variants_and_the_rest_to_none() { + fn every_llama_cpp_rope_type_converts_to_its_variant() { assert_eq!( [ + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NONE, llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NORM, llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NEOX, llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_MROPE, llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_IMROPE, llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_VISION, - llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NONE, - 9999, ] - .map(RopeType::from_raw), + .map(RopeType::try_from), [ - Some(RopeType::Norm), - Some(RopeType::NeoX), - Some(RopeType::MRope), - Some(RopeType::IMRope), - Some(RopeType::Vision), - None, - None, + Ok(RopeType::None), + Ok(RopeType::Norm), + Ok(RopeType::NeoX), + Ok(RopeType::MRope), + Ok(RopeType::IMRope), + Ok(RopeType::Vision), ] ); } + #[test] + fn a_value_llama_cpp_does_not_define_is_refused() { + assert_eq!(RopeType::try_from(9999), Err(UnknownRopeType(9999))); + } + #[test] fn only_multi_section_rope_types_place_several_positions_per_embedding() { assert_eq!( [ + RopeType::None, RopeType::Norm, RopeType::NeoX, RopeType::MRope, @@ -71,7 +83,7 @@ mod tests { RopeType::Vision, ] .map(RopeType::has_one_position_per_embedding), - [true, true, false, false, true] + [true, true, true, false, false, true] ); } } From d7f1fd1daa23b1587f90c7262d895618e31ac216 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 07:45:39 +0100 Subject: [PATCH 51/82] Return no bytes for a placeholder media bitmap instead of slicing a null pointer --- llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs index c30b16aa3..57976c734 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs @@ -369,11 +369,17 @@ impl MtmdBitmap { unsafe { llama_cpp_bindings_sys::mtmd_bitmap_get_ny(self.bitmap.as_ptr()) } } + /// The bitmap's bytes, or none for a placeholder bitmap, whose data mtmd reports as null. #[must_use] pub fn data(&self) -> &[u8] { - let ptr = unsafe { llama_cpp_bindings_sys::mtmd_bitmap_get_data(self.bitmap.as_ptr()) }; + let data = unsafe { llama_cpp_bindings_sys::mtmd_bitmap_get_data(self.bitmap.as_ptr()) }; let len = unsafe { llama_cpp_bindings_sys::mtmd_bitmap_get_n_bytes(self.bitmap.as_ptr()) }; - unsafe { slice::from_raw_parts(ptr, len) } + + if data.is_null() { + return &[]; + } + + unsafe { slice::from_raw_parts(data, len) } } #[must_use] @@ -490,6 +496,13 @@ mod tests { assert!(bitmap.is_audio()); } + #[test] + fn a_placeholder_bitmap_without_samples_holds_no_data() { + let bitmap = MtmdBitmap::from_audio_data(&[]).unwrap(); + + assert_eq!(bitmap.data(), &[] as &[u8]); + } + #[test] fn data_returns_expected_bytes_for_image() { let pixel_data: Vec = vec![255, 0, 0, 0, 255, 0, 0, 0, 255, 128, 128, 128]; From 5798c946e0493bd7d4b4013641dc6ea6e8e0813e Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 07:47:02 +0100 Subject: [PATCH 52/82] Refuse token data array selections that name no candidate in either direction --- llama-cpp-bindings/src/token/data_array.rs | 90 ++++++++++++++-------- 1 file changed, 60 insertions(+), 30 deletions(-) diff --git a/llama-cpp-bindings/src/token/data_array.rs b/llama-cpp-bindings/src/token/data_array.rs index a3285225f..3943e5402 100644 --- a/llama-cpp-bindings/src/token/data_array.rs +++ b/llama-cpp-bindings/src/token/data_array.rs @@ -47,6 +47,35 @@ impl LlamaTokenDataArray { } impl LlamaTokenDataArray { + fn c_selection(&self) -> Result { + let Some(selected_index) = self.selected else { + return Ok(-1); + }; + + self.data + .get(selected_index) + .and_then(|_selected_candidate| i64::try_from(selected_index).ok()) + .ok_or(FfiContractError { + operation: "modify_as_c_llama_token_data_array", + detail: "the token data array selects a token outside itself", + }) + } + + fn selection_of(&self, c_selected: i64) -> Result, FfiContractError> { + if c_selected == -1 { + return Ok(None); + } + + usize::try_from(c_selected) + .ok() + .filter(|&selected_index| selected_index < self.data.len()) + .map(Some) + .ok_or(FfiContractError { + operation: "modify_as_c_llama_token_data_array", + detail: "the llama.cpp sampler selected a token outside the token data array", + }) + } + /// # Errors /// /// Returns [`llama_cpp_ffi_status::ffi_contract_error::FfiContractError`] when the llama.cpp sampler grows the array beyond the @@ -71,10 +100,7 @@ impl LlamaTokenDataArray { let mut c_llama_token_data_array = llama_cpp_bindings_sys::llama_token_data_array { data, size, - selected: self - .selected - .and_then(|selected_index| selected_index.try_into().ok()) - .unwrap_or(-1), + selected: self.c_selection()?, sorted: self.sorted, }; @@ -99,11 +125,7 @@ impl LlamaTokenDataArray { } self.sorted = c_llama_token_data_array.sorted; - self.selected = c_llama_token_data_array - .selected - .try_into() - .ok() - .filter(|&selected_index| selected_index < self.data.len()); + self.selected = self.selection_of(c_llama_token_data_array.selected)?; Ok(result) } @@ -179,37 +201,45 @@ mod tests { } #[test] - fn modify_clears_selection_when_index_is_out_of_range() { + fn a_sampler_selecting_a_token_outside_the_array_is_a_contract_violation() { let mut array = LlamaTokenDataArray::new( vec![LlamaTokenData::new(LlamaToken::new(0), 1.0, 0.0)], false, ); - unsafe { - array - .modify_as_c_llama_token_data_array(|c_array| { - c_array.selected = 5; - }) - .expect("the array is left at its original size"); - } + let result = unsafe { + array.modify_as_c_llama_token_data_array(|c_array| { + c_array.selected = 5; + }) + }; - assert_eq!(array.selected, None); + assert_eq!( + result, + Err(FfiContractError { + operation: "modify_as_c_llama_token_data_array", + detail: "the llama.cpp sampler selected a token outside the token data array", + }) + ); } #[test] - fn selected_overflow_uses_negative_one() { - let mut array = LlamaTokenDataArray { - data: vec![LlamaTokenData::new(LlamaToken::new(0), 1.0, 0.0)], - selected: Some(usize::MAX), - sorted: false, - }; - - unsafe { - array - .modify_as_c_llama_token_data_array(|c_array| { - assert_eq!(c_array.selected, -1); + fn a_selection_outside_the_array_is_refused_before_reaching_llama_cpp() { + for selected in [1, usize::MAX] { + let mut array = LlamaTokenDataArray { + data: vec![LlamaTokenData::new(LlamaToken::new(0), 1.0, 0.0)], + selected: Some(selected), + sorted: false, + }; + + let result = unsafe { array.modify_as_c_llama_token_data_array(|_c_array| ()) }; + + assert_eq!( + result, + Err(FfiContractError { + operation: "modify_as_c_llama_token_data_array", + detail: "the token data array selects a token outside itself", }) - .expect("the array is left at its original size"); + ); } } From cc111939d27d5156ce15a00959ca60468f35c96e Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 08:00:55 +0100 Subject: [PATCH 53/82] Read C++ error text and media ids losslessly, refusing text that is not UTF-8 with its bytes --- .../tests/multimodal_chunk_tokenization.rs | 4 +- llama-cpp-bindings/src/chat_session.rs | 23 ++--- llama-cpp-bindings/src/chat_templates.rs | 10 ++- llama-cpp-bindings/src/context.rs | 27 ++++-- .../src/context/kv_cache_type.rs | 3 +- llama-cpp-bindings/src/context/memory.rs | 15 ++-- llama-cpp-bindings/src/context/session.rs | 11 ++- .../src/context/state_data_error.rs | 3 + .../src/error/batch_transfer_error.rs | 3 + .../src/error/chat_session_error.rs | 3 + .../src/error/chat_templates_error.rs | 3 + llama-cpp-bindings/src/error/fit_error.rs | 3 + llama-cpp-bindings/src/error/grammar_error.rs | 3 + .../src/error/json_schema_to_grammar_error.rs | 3 + .../src/error/llama_context_load_error.rs | 3 + .../src/error/llama_model_load_error.rs | 3 + .../src/error/memory_seq_add_error.rs | 3 + .../src/error/memory_seq_div_error.rs | 3 + .../src/error/memory_seq_pos_max_error.rs | 3 + .../src/error/parse_chat_message_error.rs | 3 + .../src/error/process_failure.rs | 3 + llama-cpp-bindings/src/error/sample_error.rs | 3 + .../src/error/sampler_accept_error.rs | 3 + .../src/error/sampler_apply_error.rs | 3 + .../src/error/sampling_error.rs | 3 + .../src/error/string_to_token_error.rs | 3 + .../src/json_schema_to_grammar.rs | 13 +-- .../src/lazy_grammar_triggers.rs | 18 ++-- llama-cpp-bindings/src/llama_batch.rs | 7 +- llama-cpp-bindings/src/model.rs | 10 ++- llama-cpp-bindings/src/model/params.rs | 5 +- llama-cpp-bindings/src/mtmd.rs | 1 + llama-cpp-bindings/src/mtmd/eval_chunk.rs | 5 +- llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs | 84 +++++++------------ .../src/mtmd/mtmd_bitmap_error.rs | 3 + llama-cpp-bindings/src/mtmd/mtmd_context.rs | 20 +++-- .../src/mtmd/mtmd_encode_error.rs | 3 + .../src/mtmd/mtmd_eval_error.rs | 3 + .../src/mtmd/mtmd_init_error.rs | 3 + .../src/mtmd/mtmd_input_chunk.rs | 32 ++++--- .../src/mtmd/mtmd_input_chunk_error.rs | 3 + .../src/mtmd/mtmd_input_chunks.rs | 11 ++- .../src/mtmd/mtmd_input_chunks_error.rs | 3 + .../src/mtmd/mtmd_memory_usage.rs | 11 ++- .../src/mtmd/mtmd_memory_usage_error.rs | 3 + .../src/mtmd/mtmd_tokenize_error.rs | 3 + .../src/mtmd/optional_mtmd_id.rs | 50 +++++++++++ llama-cpp-bindings/src/sampling.rs | 45 ++++++---- llama-cpp-ffi-status/src/cpp_string_error.rs | 16 ++++ llama-cpp-ffi-status/src/lib.rs | 1 + .../src/read_and_free_cpp_string.rs | 51 ++++++++--- llama-cpp-gbnf/src/gbnf_validation_error.rs | 3 + llama-cpp-gbnf/src/validate_gbnf.rs | 11 ++- 53 files changed, 407 insertions(+), 161 deletions(-) create mode 100644 llama-cpp-bindings/src/mtmd/optional_mtmd_id.rs create mode 100644 llama-cpp-ffi-status/src/cpp_string_error.rs diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs index e4745ccf7..c5b7a1887 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs @@ -151,7 +151,7 @@ fn text_chunk_id_returns_none(fixture: &LlamaFixture<'_>) -> Result<()> { let chunks = tokenize_prompt_with_synthetic_image(fixture, "Hello <__media__>"); let first_chunk = chunks.get(0).context("missing first chunk")?; assert_eq!(first_chunk.chunk_type()?, MtmdInputChunkType::Text); - assert!(first_chunk.id().is_none()); + assert_eq!(first_chunk.id(), Ok(None)); Ok(()) } @@ -207,7 +207,7 @@ fn an_image_chunk_carries_the_empty_id_of_a_bitmap_without_one( ) -> Result<()> { let chunks = tokenize_prompt_with_synthetic_image(fixture, "Hello <__media__>"); - assert_eq!(first_image_chunk(&chunks)?.id(), Some(String::new())); + assert_eq!(first_image_chunk(&chunks)?.id(), Ok(Some(String::new()))); Ok(()) } diff --git a/llama-cpp-bindings/src/chat_session.rs b/llama-cpp-bindings/src/chat_session.rs index ac2a8951a..ed9a359c6 100644 --- a/llama-cpp-bindings/src/chat_session.rs +++ b/llama-cpp-bindings/src/chat_session.rs @@ -690,6 +690,7 @@ impl Drop for ChatSession<'_> { #[cfg(test)] mod tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ffi::CStr; use std::ffi::CString; use std::ptr; @@ -813,10 +814,12 @@ mod tests { chat_session_create_status_to_result(status, ptr::null_mut(), ptr::null_mut()).err() }), refusals.map(|_status| { - Some(ChatSessionError::FfiContract(FfiContractError { - operation: "llama_rs_chat_session_create", - detail: "reported a failure without an error message", - })) + Some(ChatSessionError::CppString(CppStringError::Missing( + FfiContractError { + operation: "llama_rs_chat_session_create", + detail: "reported a failure without an error message", + }, + ))) }) ); } @@ -1131,18 +1134,18 @@ mod tests { .err(), ], [ - Some(ParseChatMessageError::FfiContract(FfiContractError { + Some(ParseChatMessageError::CppString(CppStringError::Missing(FfiContractError { operation: "llama_rs_chat_session_parse", detail: "reported a failure without an error message", - })), - Some(ParseChatMessageError::FfiContract(FfiContractError { + }))), + Some(ParseChatMessageError::CppString(CppStringError::Missing(FfiContractError { operation: "llama_rs_chat_session_parse", detail: "reported a failure without an error message", - })), - Some(ParseChatMessageError::FfiContract(FfiContractError { + }))), + Some(ParseChatMessageError::CppString(CppStringError::Missing(FfiContractError { operation: "llama_rs_chat_session_classify", detail: "reported a thrown C++ exception without an error message", - })), + }))), ] ); } diff --git a/llama-cpp-bindings/src/chat_templates.rs b/llama-cpp-bindings/src/chat_templates.rs index 4b28abe70..10c23ae1d 100644 --- a/llama-cpp-bindings/src/chat_templates.rs +++ b/llama-cpp-bindings/src/chat_templates.rs @@ -150,6 +150,7 @@ impl Drop for ChatTemplates { #[cfg(test)] mod tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ffi::c_char; use std::ptr; @@ -173,6 +174,13 @@ mod tests { }) } + fn missing_message(detail: &'static str) -> ChatTemplatesError { + ChatTemplatesError::CppString(CppStringError::Missing(FfiContractError { + operation: "llama_rs_chat_templates_init", + detail, + })) + } + #[test] fn preparing_templates_maps_every_failure_status() { assert_eq!( @@ -242,7 +250,7 @@ mod tests { llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut() ), - Some(contract( + Some(missing_message( "reported a thrown C++ exception without an error message" )) ); diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index 68cd792dc..6cb7ce090 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -754,6 +754,7 @@ mod check_lora_adapters_result_tests { #[cfg(test)] mod new_context_with_model_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -830,10 +831,10 @@ mod new_context_with_model_status_to_result_tests { assert_eq!( result, - Err(FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_new_context_with_model", detail: "reported a thrown C++ exception without an error message", - } + }) .into()) ); } @@ -926,6 +927,7 @@ mod new_context_with_model_status_to_result_tests { #[cfg(test)] mod batch_ext_init_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -1012,10 +1014,12 @@ mod batch_ext_init_status_to_result_tests { ptr::null_mut(), ) .err(), - Some(LlamaContextLoadError::FfiContract(FfiContractError { - operation: "llama_rs_batch_ext_init", - detail: "reported a thrown C++ exception without an error message", - })) + Some(LlamaContextLoadError::CppString(CppStringError::Missing( + FfiContractError { + operation: "llama_rs_batch_ext_init", + detail: "reported a thrown C++ exception without an error message", + } + ))) ); } @@ -1033,6 +1037,7 @@ mod batch_ext_init_status_to_result_tests { #[cfg(test)] mod process_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -1053,6 +1058,14 @@ mod process_status_to_result_tests { .into() } + fn missing_message(detail: &'static str) -> DecodeError { + ProcessFailure::from(CppStringError::Missing(FfiContractError { + operation: OPERATION, + detail, + })) + .into() + } + #[test] fn every_status_without_a_return_code_or_message_maps_to_its_result() { let statuses = [ @@ -1081,7 +1094,7 @@ mod process_status_to_result_tests { )), Err(ProcessFailure::NotEnoughMemory.into()), Err(ProcessFailure::LlamaCppOutOfMemory.into()), - Err(contract_violation( + Err(missing_message( "reported a thrown C++ exception without an error message" )), Err(contract_violation("was given a null ctx argument")), diff --git a/llama-cpp-bindings/src/context/kv_cache_type.rs b/llama-cpp-bindings/src/context/kv_cache_type.rs index e43fdd230..4f8d4389c 100644 --- a/llama-cpp-bindings/src/context/kv_cache_type.rs +++ b/llama-cpp-bindings/src/context/kv_cache_type.rs @@ -143,7 +143,8 @@ mod tests { let upstream_name = unsafe { CStr::from_ptr(llama_cpp_bindings_sys::ggml_type_name(raw)) } - .to_string_lossy(); + .to_str() + .expect("ggml type names are ASCII"); assert_ne!( KvCacheType::from(raw), diff --git a/llama-cpp-bindings/src/context/memory.rs b/llama-cpp-bindings/src/context/memory.rs index 6ee0f2d89..96152dcf5 100644 --- a/llama-cpp-bindings/src/context/memory.rs +++ b/llama-cpp-bindings/src/context/memory.rs @@ -343,6 +343,7 @@ impl LlamaContext<'_> { #[cfg(test)] mod memory_seq_add_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -391,10 +392,10 @@ mod memory_seq_add_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), ), - Err(FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_memory_seq_add", detail: "reported a thrown C++ exception without an error message", - } + }) .into()) ); } @@ -458,6 +459,7 @@ mod memory_seq_add_status_to_result_tests { #[cfg(test)] mod memory_seq_div_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -506,10 +508,10 @@ mod memory_seq_div_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), ), - Err(FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_memory_seq_div", detail: "reported a thrown C++ exception without an error message", - } + }) .into()) ); } @@ -573,6 +575,7 @@ mod memory_seq_div_status_to_result_tests { #[cfg(test)] mod memory_seq_pos_max_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -670,10 +673,10 @@ mod memory_seq_pos_max_status_to_result_tests { -1, ptr::null_mut(), ), - Err(FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_memory_seq_pos_max", detail: "reported a thrown C++ exception without an error message", - } + }) .into()) ); } diff --git a/llama-cpp-bindings/src/context/session.rs b/llama-cpp-bindings/src/context/session.rs index f0aea554f..ab5037c7f 100644 --- a/llama-cpp-bindings/src/context/session.rs +++ b/llama-cpp-bindings/src/context/session.rs @@ -487,6 +487,7 @@ mod process_seq_load_result_tests { #[cfg(test)] mod state_data_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -577,10 +578,12 @@ mod state_data_status_to_result_tests { ptr::null_mut(), "llama_rs_state_get_data", ), - Err(StateDataError::FfiContract(FfiContractError { - operation: "llama_rs_state_get_data", - detail: "reported a thrown C++ exception without an error message", - })) + Err(StateDataError::CppString(CppStringError::Missing( + FfiContractError { + operation: "llama_rs_state_get_data", + detail: "reported a thrown C++ exception without an error message", + } + ))) ); } diff --git a/llama-cpp-bindings/src/context/state_data_error.rs b/llama-cpp-bindings/src/context/state_data_error.rs index 3cd41a60c..10355dfe5 100644 --- a/llama-cpp-bindings/src/context/state_data_error.rs +++ b/llama-cpp-bindings/src/context/state_data_error.rs @@ -1,8 +1,11 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum StateDataError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/error/batch_transfer_error.rs b/llama-cpp-bindings/src/error/batch_transfer_error.rs index 535aa5a55..ae766d17c 100644 --- a/llama-cpp-bindings/src/error/batch_transfer_error.rs +++ b/llama-cpp-bindings/src/error/batch_transfer_error.rs @@ -1,3 +1,4 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; @@ -5,6 +6,8 @@ use crate::context::context_limits::ContextLimits; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum BatchTransferError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/error/chat_session_error.rs b/llama-cpp-bindings/src/error/chat_session_error.rs index 04d179703..e3689448c 100644 --- a/llama-cpp-bindings/src/error/chat_session_error.rs +++ b/llama-cpp-bindings/src/error/chat_session_error.rs @@ -1,6 +1,7 @@ use std::ffi::NulError; use std::str::Utf8Error; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; @@ -11,6 +12,8 @@ use crate::error::string_to_token_error::StringToTokenError; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum ChatSessionError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/error/chat_templates_error.rs b/llama-cpp-bindings/src/error/chat_templates_error.rs index a8edbcd7a..289c5e535 100644 --- a/llama-cpp-bindings/src/error/chat_templates_error.rs +++ b/llama-cpp-bindings/src/error/chat_templates_error.rs @@ -1,8 +1,11 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum ChatTemplatesError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/error/fit_error.rs b/llama-cpp-bindings/src/error/fit_error.rs index 341bff51f..c6a61377b 100644 --- a/llama-cpp-bindings/src/error/fit_error.rs +++ b/llama-cpp-bindings/src/error/fit_error.rs @@ -1,8 +1,11 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] pub enum FitError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/error/grammar_error.rs b/llama-cpp-bindings/src/error/grammar_error.rs index 68481c528..9a67fc7f2 100644 --- a/llama-cpp-bindings/src/error/grammar_error.rs +++ b/llama-cpp-bindings/src/error/grammar_error.rs @@ -1,5 +1,6 @@ use std::ffi::NulError; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; @@ -8,6 +9,8 @@ use crate::error::token_to_string_error::TokenToStringError; #[derive(Debug, thiserror::Error, PartialEq, Eq)] pub enum GrammarError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/error/json_schema_to_grammar_error.rs b/llama-cpp-bindings/src/error/json_schema_to_grammar_error.rs index be4aa8373..ff1319fd4 100644 --- a/llama-cpp-bindings/src/error/json_schema_to_grammar_error.rs +++ b/llama-cpp-bindings/src/error/json_schema_to_grammar_error.rs @@ -1,11 +1,14 @@ use std::ffi::NulError; use std::string::FromUtf8Error; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; #[derive(Debug, PartialEq, Eq, thiserror::Error)] pub enum JsonSchemaToGrammarError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/error/llama_context_load_error.rs b/llama-cpp-bindings/src/error/llama_context_load_error.rs index 8fed19da9..efd6c7671 100644 --- a/llama-cpp-bindings/src/error/llama_context_load_error.rs +++ b/llama-cpp-bindings/src/error/llama_context_load_error.rs @@ -1,3 +1,4 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; @@ -6,6 +7,8 @@ use crate::error::unknown_pooling_type::UnknownPoolingType; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum LlamaContextLoadError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/error/llama_model_load_error.rs b/llama-cpp-bindings/src/error/llama_model_load_error.rs index 7f9081996..340e8fcab 100644 --- a/llama-cpp-bindings/src/error/llama_model_load_error.rs +++ b/llama-cpp-bindings/src/error/llama_model_load_error.rs @@ -1,11 +1,14 @@ use std::ffi::NulError; use std::path::PathBuf; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; #[derive(Debug, PartialEq, Eq, thiserror::Error)] pub enum LlamaModelLoadError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/error/memory_seq_add_error.rs b/llama-cpp-bindings/src/error/memory_seq_add_error.rs index d81b4bfdb..1040a5cd3 100644 --- a/llama-cpp-bindings/src/error/memory_seq_add_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_add_error.rs @@ -1,5 +1,6 @@ use std::num::TryFromIntError; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; @@ -8,6 +9,8 @@ use crate::error::unknown_rope_type::UnknownRopeType; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum MemorySeqAddError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/error/memory_seq_div_error.rs b/llama-cpp-bindings/src/error/memory_seq_div_error.rs index 9a80bcd43..5aa855023 100644 --- a/llama-cpp-bindings/src/error/memory_seq_div_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_div_error.rs @@ -1,5 +1,6 @@ use std::num::TryFromIntError; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; @@ -8,6 +9,8 @@ use crate::error::unknown_rope_type::UnknownRopeType; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum MemorySeqDivError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs b/llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs index 7bf1601c1..44f040c04 100644 --- a/llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs @@ -1,3 +1,4 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; @@ -5,6 +6,8 @@ use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum MemorySeqPosMaxError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiContract(#[from] FfiContractError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/error/parse_chat_message_error.rs b/llama-cpp-bindings/src/error/parse_chat_message_error.rs index 5dae9bccc..7ecfd4852 100644 --- a/llama-cpp-bindings/src/error/parse_chat_message_error.rs +++ b/llama-cpp-bindings/src/error/parse_chat_message_error.rs @@ -1,10 +1,13 @@ use std::str::Utf8Error; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum ParseChatMessageError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/error/process_failure.rs b/llama-cpp-bindings/src/error/process_failure.rs index 9433b598c..c5f2663cd 100644 --- a/llama-cpp-bindings/src/error/process_failure.rs +++ b/llama-cpp-bindings/src/error/process_failure.rs @@ -1,9 +1,12 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; /// How `llama_rs_process` failed before llama.cpp returned a code of its own. #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum ProcessFailure { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/error/sample_error.rs b/llama-cpp-bindings/src/error/sample_error.rs index 7be976fb1..7b1a1960c 100644 --- a/llama-cpp-bindings/src/error/sample_error.rs +++ b/llama-cpp-bindings/src/error/sample_error.rs @@ -1,3 +1,4 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; @@ -7,6 +8,8 @@ use crate::error::sampler_apply_error::SamplerApplyError; #[derive(Debug, thiserror::Error, PartialEq, Eq)] pub enum SampleError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/error/sampler_accept_error.rs b/llama-cpp-bindings/src/error/sampler_accept_error.rs index 91331b6a9..cb7e590b4 100644 --- a/llama-cpp-bindings/src/error/sampler_accept_error.rs +++ b/llama-cpp-bindings/src/error/sampler_accept_error.rs @@ -1,8 +1,11 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; #[derive(Debug, PartialEq, Eq, thiserror::Error)] pub enum SamplerAcceptError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/error/sampler_apply_error.rs b/llama-cpp-bindings/src/error/sampler_apply_error.rs index d7ed80125..daddb2e61 100644 --- a/llama-cpp-bindings/src/error/sampler_apply_error.rs +++ b/llama-cpp-bindings/src/error/sampler_apply_error.rs @@ -1,8 +1,11 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; #[derive(Debug, thiserror::Error, Clone, PartialEq, Eq)] pub enum SamplerApplyError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/error/sampling_error.rs b/llama-cpp-bindings/src/error/sampling_error.rs index 6b70a3229..9c5631fa4 100644 --- a/llama-cpp-bindings/src/error/sampling_error.rs +++ b/llama-cpp-bindings/src/error/sampling_error.rs @@ -1,10 +1,13 @@ use std::ffi::NulError; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum SamplingError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/error/string_to_token_error.rs b/llama-cpp-bindings/src/error/string_to_token_error.rs index 93d55a66f..082f33dd2 100644 --- a/llama-cpp-bindings/src/error/string_to_token_error.rs +++ b/llama-cpp-bindings/src/error/string_to_token_error.rs @@ -1,10 +1,13 @@ use std::ffi::NulError; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; #[derive(Debug, PartialEq, Eq, thiserror::Error)] pub enum StringToTokenError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/json_schema_to_grammar.rs b/llama-cpp-bindings/src/json_schema_to_grammar.rs index 10ad02ff1..79aeb94c2 100644 --- a/llama-cpp-bindings/src/json_schema_to_grammar.rs +++ b/llama-cpp-bindings/src/json_schema_to_grammar.rs @@ -184,6 +184,7 @@ mod tests { #[cfg(test)] mod json_schema_to_grammar_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ffi::c_char; use std::ptr; @@ -205,10 +206,10 @@ mod json_schema_to_grammar_status_to_result_tests { assert_eq!( result, - Err(FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "reported a thrown C++ exception without an error message", - } + }) .into()) ); } @@ -225,10 +226,10 @@ mod json_schema_to_grammar_status_to_result_tests { assert_eq!( result, - Err(FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "reported a thrown C++ exception without an error message", - } + }) .into()) ); } @@ -245,10 +246,10 @@ mod json_schema_to_grammar_status_to_result_tests { assert_eq!( result, - Err(FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "reported a thrown C++ exception without an error message", - } + }) .into()) ); } diff --git a/llama-cpp-bindings/src/lazy_grammar_triggers.rs b/llama-cpp-bindings/src/lazy_grammar_triggers.rs index 86213f8ed..6dfae11a4 100644 --- a/llama-cpp-bindings/src/lazy_grammar_triggers.rs +++ b/llama-cpp-bindings/src/lazy_grammar_triggers.rs @@ -125,6 +125,7 @@ impl LazyGrammarTriggers { #[cfg(test)] mod tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_fault_injection::armed_fault::ArmedFault; @@ -199,7 +200,12 @@ mod tests { ], [ Ok("\\[TOOL\\]".to_owned()), - contract("success status contained a null string"), + Err(GrammarError::CppString(CppStringError::Missing( + FfiContractError { + operation: "llama_rs_regex_escape", + detail: "success status contained a null string", + } + ))), Err(GrammarError::Reported { message: "the escape threw".to_owned() }), @@ -224,10 +230,12 @@ mod tests { ptr::null_mut(), ptr::null_mut() ), - Err(GrammarError::FfiContract(FfiContractError { - operation: "llama_rs_regex_escape", - detail: "reported a thrown C++ exception without an error message" - })) + Err(GrammarError::CppString(CppStringError::Missing( + FfiContractError { + operation: "llama_rs_regex_escape", + detail: "reported a thrown C++ exception without an error message" + } + ))) ); } diff --git a/llama-cpp-bindings/src/llama_batch.rs b/llama-cpp-bindings/src/llama_batch.rs index d953749a9..9c5e8e710 100644 --- a/llama-cpp-bindings/src/llama_batch.rs +++ b/llama-cpp-bindings/src/llama_batch.rs @@ -429,6 +429,7 @@ mod tests { #[cfg(test)] mod batch_ext_assign_entries_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -556,12 +557,10 @@ mod batch_ext_assign_entries_status_to_result_tests { 0, ptr::null_mut(), ), - Err(BatchTransferError::FfiContract( - FfiContractError { + Err(BatchTransferError::CppString(CppStringError::Missing(FfiContractError { operation: "llama_rs_batch_ext_assign_entries", detail: "reported a thrown C++ exception without an error message", - } - )) + }))) ); } diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index f52bed550..d443a48fb 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -1157,6 +1157,7 @@ mod initialize_lora_adapter_tests { #[cfg(test)] mod load_model_from_file_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::path::Path; use std::ptr; @@ -1253,10 +1254,10 @@ mod load_model_from_file_status_to_result_tests { assert_eq!( result.unwrap_err(), - FfiContractError { + CppStringError::Missing(FfiContractError { operation: "llama_rs_load_model_from_file", detail: "reported a thrown C++ exception without an error message", - } + }) .into() ); } @@ -1368,6 +1369,7 @@ mod load_model_from_file_status_to_result_tests { #[cfg(test)] mod tokenize_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -1434,10 +1436,10 @@ mod tokenize_status_to_result_tests { assert_eq!( result, - Err(FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_tokenize", detail: "reported a thrown C++ exception without an error message", - } + }) .into()) ); } diff --git a/llama-cpp-bindings/src/model/params.rs b/llama-cpp-bindings/src/model/params.rs index 4e1f11392..79f2805ff 100644 --- a/llama-cpp-bindings/src/model/params.rs +++ b/llama-cpp-bindings/src/model/params.rs @@ -917,6 +917,7 @@ mod tests { #[cfg(test)] mod fit_params_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -1022,10 +1023,10 @@ mod fit_params_status_to_result_tests { assert_eq!( result, - Err(FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_fit_params", detail: "reported a thrown C++ exception without an error message", - } + }) .into()) ); } diff --git a/llama-cpp-bindings/src/mtmd.rs b/llama-cpp-bindings/src/mtmd.rs index 0ef3d45e5..f615b58a2 100644 --- a/llama-cpp-bindings/src/mtmd.rs +++ b/llama-cpp-bindings/src/mtmd.rs @@ -1,5 +1,6 @@ mod eval_chunk; mod mtmd_ffi_input_part; +mod optional_mtmd_id; pub mod fitted_mtmd_input_chunk; pub mod fitted_mtmd_input_chunks; diff --git a/llama-cpp-bindings/src/mtmd/eval_chunk.rs b/llama-cpp-bindings/src/mtmd/eval_chunk.rs index 25a686bb2..1b1cb8163 100644 --- a/llama-cpp-bindings/src/mtmd/eval_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/eval_chunk.rs @@ -118,6 +118,7 @@ pub fn eval_chunk( #[cfg(test)] mod eval_chunk_single_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -199,10 +200,10 @@ mod eval_chunk_single_status_to_result_tests { assert_eq!( result, - Err(FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_mtmd_eval_chunk_single", detail: "reported a thrown C++ exception without an error message", - } + }) .into()) ); } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs index 57976c734..0302807a1 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs @@ -1,9 +1,10 @@ use std::ffi::NulError; -use std::ffi::{CStr, CString, c_char}; +use std::ffi::{CString, c_char}; use std::path::PathBuf; use std::ptr; use std::ptr::NonNull; use std::slice; +use std::str::Utf8Error; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; @@ -11,18 +12,7 @@ use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use super::mtmd_bitmap_error::MtmdBitmapError; use super::mtmd_context::MtmdContext; - -fn cstr_ptr_to_optional_string(ptr: *const c_char) -> Option { - if ptr.is_null() { - None - } else { - let id = unsafe { CStr::from_ptr(ptr) } - .to_string_lossy() - .into_owned(); - - Some(id) - } -} +use super::optional_mtmd_id::optional_mtmd_id; /// # Safety /// @@ -387,11 +377,15 @@ impl MtmdBitmap { unsafe { llama_cpp_bindings_sys::mtmd_bitmap_is_audio(self.bitmap.as_ptr()) } } - #[must_use] - pub fn id(&self) -> Option { - let ptr = unsafe { llama_cpp_bindings_sys::mtmd_bitmap_get_id(self.bitmap.as_ptr()) }; - - cstr_ptr_to_optional_string(ptr) + /// # Errors + /// + /// Returns [`Utf8Error`] when the id is not valid UTF-8. + pub fn id(&self) -> Result, Utf8Error> { + unsafe { + optional_mtmd_id(llama_cpp_bindings_sys::mtmd_bitmap_get_id( + self.bitmap.as_ptr(), + )) + } } /// # Errors @@ -483,7 +477,7 @@ mod tests { let bitmap = MtmdBitmap::from_image_data(2, 2, &image_data).unwrap(); bitmap.set_id("test_image").unwrap(); - assert_eq!(bitmap.id().as_deref(), Some("test_image")); + assert_eq!(bitmap.id(), Ok(Some("test_image".to_owned()))); } #[test] @@ -517,7 +511,7 @@ mod tests { let image_data = vec![0u8; 12]; let bitmap = MtmdBitmap::from_image_data(2, 2, &image_data).unwrap(); - assert_eq!(bitmap.id(), Some(String::new())); + assert_eq!(bitmap.id(), Ok(Some(String::new()))); } #[test] @@ -530,29 +524,9 @@ mod tests { } } -#[cfg(test)] -mod cstr_ptr_to_optional_string_tests { - use std::ffi::CString; - use std::ptr; - - use super::cstr_ptr_to_optional_string; - - #[test] - fn cstr_ptr_to_optional_string_returns_none_for_null() { - assert_eq!(cstr_ptr_to_optional_string(ptr::null()), None); - } - - #[test] - fn cstr_ptr_to_optional_string_copies_a_non_null_string() { - let cstr = CString::new("hello").unwrap(); - let result = cstr_ptr_to_optional_string(cstr.as_ptr()); - - assert_eq!(result, Some("hello".to_string())); - } -} - #[cfg(test)] mod from_file_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::path::PathBuf; use std::ptr; @@ -628,10 +602,10 @@ mod from_file_status_to_result_tests { assert_eq!( result.unwrap_err(), - FfiContractError { + CppStringError::Missing(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "reported a thrown C++ exception without an error message", - } + }) .into() ); } @@ -754,6 +728,7 @@ mod from_file_status_to_result_tests { #[cfg(test)] mod from_buf_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -859,10 +834,10 @@ mod from_buf_status_to_result_tests { assert_eq!( result.err(), Some( - FfiContractError { + CppStringError::Missing(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail: "reported a thrown C++ exception without an error message", - } + }) .into() ) ); @@ -950,6 +925,7 @@ mod from_buf_status_to_result_tests { #[cfg(test)] mod bitmap_init_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -963,6 +939,14 @@ mod bitmap_init_status_to_result_tests { FfiContractError { operation, detail }.into() } + fn missing_message(operation: &'static str) -> MtmdBitmapError { + CppStringError::Missing(FfiContractError { + operation, + detail: "reported a thrown C++ exception without an error message", + }) + .into() + } + #[test] fn every_image_failure_status_maps_to_its_error() { let statuses = [ @@ -995,10 +979,7 @@ mod bitmap_init_status_to_result_tests { )), Some(MtmdBitmapError::NotEnoughMemory), Some(MtmdBitmapError::LlamaCppOutOfMemory), - Some(contract_violation( - operation, - "reported a thrown C++ exception without an error message" - )), + Some(missing_message(operation)), Some(MtmdBitmapError::FfiStatus(FfiStatusError { operation, code: 255, @@ -1040,10 +1021,7 @@ mod bitmap_init_status_to_result_tests { )), Some(MtmdBitmapError::NotEnoughMemory), Some(MtmdBitmapError::LlamaCppOutOfMemory), - Some(contract_violation( - operation, - "reported a thrown C++ exception without an error message" - )), + Some(missing_message(operation)), Some(MtmdBitmapError::FfiStatus(FfiStatusError { operation, code: 255, diff --git a/llama-cpp-bindings/src/mtmd/mtmd_bitmap_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_bitmap_error.rs index 5429c14b9..c0487d2bd 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_bitmap_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_bitmap_error.rs @@ -1,10 +1,13 @@ use std::path::PathBuf; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdBitmapError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_context.rs b/llama-cpp-bindings/src/mtmd/mtmd_context.rs index ec12ef9f7..cfaf54bef 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_context.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_context.rs @@ -476,6 +476,7 @@ impl Drop for MtmdContext { #[cfg(test)] mod map_tokenize_from_parts_status_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -618,10 +619,10 @@ mod map_tokenize_from_parts_status_tests { ) .err(), Some( - FfiContractError { + CppStringError::Missing(FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "reported a thrown C++ exception without an error message", - } + }) .into() ) ); @@ -699,6 +700,7 @@ mod map_tokenize_from_parts_status_tests { #[cfg(test)] mod map_tokenize_status_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -778,10 +780,10 @@ mod map_tokenize_status_tests { assert_eq!( result, - Err(FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "reported a thrown C++ exception without an error message", - } + }) .into()) ); } @@ -903,6 +905,7 @@ mod map_tokenize_status_tests { #[cfg(test)] mod map_encode_chunk_status_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -954,10 +957,10 @@ mod map_encode_chunk_status_tests { assert_eq!( result, - Err(FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_mtmd_encode_chunk", detail: "reported a thrown C++ exception without an error message", - } + }) .into()) ); } @@ -1043,6 +1046,7 @@ mod map_encode_chunk_status_tests { #[cfg(test)] mod map_init_from_file_status_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -1092,10 +1096,10 @@ mod map_init_from_file_status_tests { assert_eq!( result.unwrap_err(), - FfiContractError { + CppStringError::Missing(FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "reported a thrown C++ exception without an error message", - } + }) .into() ); } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_encode_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_encode_error.rs index c2f3efc3c..e76405dc8 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_encode_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_encode_error.rs @@ -1,8 +1,11 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdEncodeError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs index 9f3ac9e07..dab1eb4bd 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs @@ -1,3 +1,4 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; @@ -7,6 +8,8 @@ use crate::mtmd::non_causal_chunk_micro_batch_mismatch::NonCausalChunkMicroBatch #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdEvalError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_init_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_init_error.rs index d4b0ce4a4..9fac767e0 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_init_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_init_error.rs @@ -1,5 +1,6 @@ use std::path::PathBuf; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; @@ -7,6 +8,8 @@ use super::mtmd_context_params_error::MtmdContextParamsError; #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdInitError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs index 52e57c955..b09ac0f86 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs @@ -1,8 +1,8 @@ -use std::ffi::CStr; use std::ffi::c_char; use std::ptr; use std::ptr::NonNull; use std::slice; +use std::str::Utf8Error; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; @@ -14,6 +14,7 @@ use super::mtmd_input_chunk_error::MtmdInputChunkError; use super::mtmd_input_chunk_type::MtmdInputChunkType; use super::mtmd_input_chunk_type_error::MtmdInputChunkTypeError; use super::non_causal_chunk_micro_batch_mismatch::NonCausalChunkMicroBatchMismatch; +use super::optional_mtmd_id::optional_mtmd_id; use crate::token::LlamaToken; /// # Safety @@ -154,16 +155,14 @@ impl MtmdInputChunk { unsafe { llama_cpp_bindings_sys::mtmd_input_chunk_get_n_pos(self.chunk.as_ptr()) } } - #[must_use] - pub fn id(&self) -> Option { - let ptr = unsafe { llama_cpp_bindings_sys::mtmd_input_chunk_get_id(self.chunk.as_ptr()) }; - if ptr.is_null() { - None - } else { - unsafe { CStr::from_ptr(ptr) } - .to_string_lossy() - .into_owned() - .into() + /// # Errors + /// + /// Returns [`Utf8Error`] when the id is not valid UTF-8. + pub fn id(&self) -> Result, Utf8Error> { + unsafe { + optional_mtmd_id(llama_cpp_bindings_sys::mtmd_input_chunk_get_id( + self.chunk.as_ptr(), + )) } } @@ -279,6 +278,7 @@ mod non_causal_chunk_micro_batch_error_tests { #[cfg(test)] mod input_chunk_copy_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -297,6 +297,14 @@ mod input_chunk_copy_status_to_result_tests { .into() } + fn missing_message(detail: &'static str) -> MtmdInputChunkError { + CppStringError::Missing(FfiContractError { + operation: OPERATION, + detail, + }) + .into() + } + #[test] fn every_failure_status_maps_to_its_error() { let statuses = [ @@ -319,7 +327,7 @@ mod input_chunk_copy_status_to_result_tests { Some(contract_violation("was given a null out_chunk argument")), Some(MtmdInputChunkError::NotEnoughMemory), Some(MtmdInputChunkError::LlamaCppOutOfMemory), - Some(contract_violation( + Some(missing_message( "reported a thrown C++ exception without an error message" )), Some(MtmdInputChunkError::FfiStatus(FfiStatusError { diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_error.rs index 928d551a0..70a01ba8b 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_error.rs @@ -1,8 +1,11 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdInputChunkError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs index a79a82bcd..10552a150 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs @@ -238,6 +238,7 @@ mod tests { #[cfg(test)] mod input_chunks_init_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -256,6 +257,14 @@ mod input_chunks_init_status_to_result_tests { .into() } + fn missing_message(detail: &'static str) -> MtmdInputChunksError { + CppStringError::Missing(FfiContractError { + operation: OPERATION, + detail, + }) + .into() + } + #[test] fn every_failure_status_maps_to_its_error() { let statuses = [ @@ -276,7 +285,7 @@ mod input_chunks_init_status_to_result_tests { Some(contract_violation("was given a null out_chunks argument")), Some(MtmdInputChunksError::NotEnoughMemory), Some(MtmdInputChunksError::LlamaCppOutOfMemory), - Some(contract_violation( + Some(missing_message( "reported a thrown C++ exception without an error message" )), Some(MtmdInputChunksError::FfiStatus(FfiStatusError { diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks_error.rs index acd451eea..45c2bad0f 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks_error.rs @@ -1,8 +1,11 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdInputChunksError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs index 21340f270..6c6923cda 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs @@ -178,6 +178,7 @@ impl MtmdMemoryUsage { #[cfg(test)] mod tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::num::NonZeroU32; use std::ptr; @@ -229,6 +230,14 @@ mod tests { .into() } + fn missing_message(detail: &'static str) -> MtmdMemoryUsageError { + CppStringError::Missing(FfiContractError { + operation: "llama_rs_mtmd_get_memory_usage", + detail, + }) + .into() + } + #[test] fn measuring_a_missing_projector_reports_it_unloadable() { assert_eq!( @@ -420,7 +429,7 @@ mod tests { llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), ), - Err(contract_violation( + Err(missing_message( "reported a thrown C++ exception without an error message" )) ); diff --git a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage_error.rs index e5983ecc4..fce538d5e 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage_error.rs @@ -1,5 +1,6 @@ use std::ffi::CString; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; @@ -7,6 +8,8 @@ use super::mtmd_context_params_error::MtmdContextParamsError; #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdMemoryUsageError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_tokenize_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_tokenize_error.rs index b67026c0c..9f90570c8 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_tokenize_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_tokenize_error.rs @@ -1,3 +1,4 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; @@ -5,6 +6,8 @@ use crate::mtmd::mtmd_input_chunks_error::MtmdInputChunksError; #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdTokenizeError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-bindings/src/mtmd/optional_mtmd_id.rs b/llama-cpp-bindings/src/mtmd/optional_mtmd_id.rs new file mode 100644 index 000000000..ffb1ca4f9 --- /dev/null +++ b/llama-cpp-bindings/src/mtmd/optional_mtmd_id.rs @@ -0,0 +1,50 @@ +use std::ffi::CStr; +use std::ffi::c_char; +use std::str::Utf8Error; + +/// The id mtmd reports for a bitmap or a chunk, or `None` when mtmd reports no id. +/// +/// # Errors +/// +/// Returns [`Utf8Error`] when the id is not valid UTF-8. +/// +/// # Safety +/// +/// `id` must be null or point to a NUL-terminated string that outlives the call. +pub unsafe fn optional_mtmd_id(id: *const c_char) -> Result, Utf8Error> { + if id.is_null() { + return Ok(None); + } + + Ok(Some(unsafe { CStr::from_ptr(id) }.to_str()?.to_owned())) +} + +#[cfg(test)] +mod tests { + use std::ptr; + + use super::optional_mtmd_id; + + #[test] + fn a_missing_id_is_none() { + assert_eq!(unsafe { optional_mtmd_id(ptr::null()) }, Ok(None)); + } + + #[test] + fn an_id_is_copied() { + assert_eq!( + unsafe { optional_mtmd_id(c"image".as_ptr()) }, + Ok(Some("image".to_owned())) + ); + } + + #[test] + fn an_id_that_is_not_utf8_is_refused() { + let not_utf8 = c"\xff"; + + assert_eq!( + unsafe { optional_mtmd_id(not_utf8.as_ptr()) }, + Err(not_utf8.to_str().expect_err("0xff is not UTF-8")) + ); + } +} diff --git a/llama-cpp-bindings/src/sampling.rs b/llama-cpp-bindings/src/sampling.rs index 108e2ab41..03d344d8c 100644 --- a/llama-cpp-bindings/src/sampling.rs +++ b/llama-cpp-bindings/src/sampling.rs @@ -1348,6 +1348,7 @@ mod grammar_callback_error_to_accept_result_tests { #[cfg(test)] mod check_sampler_accept_status_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -1442,10 +1443,12 @@ mod check_sampler_accept_status_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), ), - Err(SamplerAcceptError::FfiContract(FfiContractError { - operation: "llama_rs_sampler_accept", - detail: "reported a thrown C++ exception without an error message", - })) + Err(SamplerAcceptError::CppString(CppStringError::Missing( + FfiContractError { + operation: "llama_rs_sampler_accept", + detail: "reported a thrown C++ exception without an error message", + } + ))) ); } @@ -1469,6 +1472,7 @@ mod check_sampler_accept_status_tests { #[cfg(test)] mod sampler_sample_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -1498,10 +1502,10 @@ mod sampler_sample_status_to_result_tests { assert_eq!( result.unwrap_err(), - FfiContractError { + CppStringError::Missing(FfiContractError { operation: "llama_rs_sampler_sample", detail: "reported a thrown C++ exception without an error message", - } + }) .into() ); } @@ -1625,6 +1629,7 @@ mod sampler_sample_status_to_result_tests { #[cfg(test)] mod sampler_init_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_fault_injection::armed_fault::ArmedFault; @@ -1754,10 +1759,10 @@ mod sampler_init_status_to_result_tests { ) .err(), Some( - FfiContractError { + CppStringError::Missing(FfiContractError { operation: "llama_rs_sampler_init", detail: "reported a thrown C++ exception without an error message", - } + }) .into() ) ); @@ -1821,6 +1826,7 @@ mod sampler_init_status_to_result_tests { #[cfg(test)] mod sampler_chain_add_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -1921,10 +1927,10 @@ mod sampler_chain_add_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), ), - Err(FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "reported a thrown C++ exception without an error message", - } + }) .into()) ); } @@ -1951,6 +1957,7 @@ mod sampler_chain_add_status_to_result_tests { #[cfg(test)] mod sampler_init_grammar_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -1991,10 +1998,10 @@ mod sampler_init_grammar_status_to_result_tests { assert_eq!( result.unwrap_err(), - FfiContractError { + CppStringError::Missing(FfiContractError { operation: "llama_rs_sampler_init_grammar", detail: "reported a thrown C++ exception without an error message", - } + }) .into() ); } @@ -2085,6 +2092,7 @@ mod sampler_init_grammar_status_to_result_tests { #[cfg(test)] mod sampler_init_grammar_lazy_patterns_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -2125,10 +2133,10 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { assert_eq!( result.unwrap_err(), - FfiContractError { + CppStringError::Missing(FfiContractError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", detail: "reported a thrown C++ exception without an error message", - } + }) .into() ); } @@ -2143,10 +2151,10 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { ) .err(), Some( - FfiContractError { + CppStringError::Missing(FfiContractError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", detail: "reported a thrown C++ exception without an error message", - } + }) .into() ) ); @@ -2325,6 +2333,7 @@ mod sampler_application_tests { #[cfg(test)] mod sampler_apply_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; @@ -2351,10 +2360,10 @@ mod sampler_apply_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), ), - Err(FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_sampler_apply", detail: "reported a thrown C++ exception without an error message", - } + }) .into()), ); } diff --git a/llama-cpp-ffi-status/src/cpp_string_error.rs b/llama-cpp-ffi-status/src/cpp_string_error.rs new file mode 100644 index 000000000..ed1b355fd --- /dev/null +++ b/llama-cpp-ffi-status/src/cpp_string_error.rs @@ -0,0 +1,16 @@ +use std::str::Utf8Error; + +use crate::ffi_contract_error::FfiContractError; + +#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)] +pub enum CppStringError { + #[error(transparent)] + Missing(#[from] FfiContractError), + #[error("{operation} reported text that is not valid UTF-8: {source}")] + NotUtf8 { + operation: &'static str, + bytes: Vec, + #[source] + source: Utf8Error, + }, +} diff --git a/llama-cpp-ffi-status/src/lib.rs b/llama-cpp-ffi-status/src/lib.rs index 84bbf604a..473752b16 100644 --- a/llama-cpp-ffi-status/src/lib.rs +++ b/llama-cpp-ffi-status/src/lib.rs @@ -8,6 +8,7 @@ ) )] +pub mod cpp_string_error; pub mod ffi_contract_error; pub mod ffi_status_error; pub mod read_and_free_cpp_string; diff --git a/llama-cpp-ffi-status/src/read_and_free_cpp_string.rs b/llama-cpp-ffi-status/src/read_and_free_cpp_string.rs index 9a92bf2ba..e5bd2aa08 100644 --- a/llama-cpp-ffi-status/src/read_and_free_cpp_string.rs +++ b/llama-cpp-ffi-status/src/read_and_free_cpp_string.rs @@ -1,16 +1,26 @@ use std::ffi::CStr; use std::ffi::c_char; +use crate::cpp_string_error::CppStringError; use crate::ffi_contract_error::FfiContractError; +fn utf8_text_of(bytes: Vec, operation: &'static str) -> Result { + String::from_utf8(bytes).map_err(|not_utf8| CppStringError::NotUtf8 { + operation, + source: not_utf8.utf8_error(), + bytes: not_utf8.into_bytes(), + }) +} + /// Takes ownership of a C string a wrapper stored in one of its out-parameters and /// hands the allocation back to the wrapper. /// /// # Errors /// -/// Returns [`FfiContractError`] when `string_ptr` is null. A wrapper always fills the -/// slot it says it filled, so a null pointer means the wrapper broke its own contract -/// rather than that the string is unknown. +/// Returns [`CppStringError::Missing`] when `string_ptr` is null. A wrapper always fills the +/// slot it says it filled, so a null pointer means the wrapper broke its own contract rather +/// than that the string is unknown. Returns [`CppStringError::NotUtf8`], keeping the bytes, +/// when the string is not valid UTF-8. /// /// # Safety /// @@ -20,21 +30,20 @@ pub unsafe fn read_and_free_cpp_string( string_ptr: *mut c_char, operation: &'static str, detail_when_missing: &'static str, -) -> Result { +) -> Result { if string_ptr.is_null() { return Err(FfiContractError { operation, detail: detail_when_missing, - }); + } + .into()); } - let value = unsafe { CStr::from_ptr(string_ptr) } - .to_string_lossy() - .into_owned(); + let bytes = unsafe { CStr::from_ptr(string_ptr) }.to_bytes().to_vec(); unsafe { llama_cpp_bindings_sys::llama_rs_string_free(string_ptr) }; - Ok(value) + utf8_text_of(bytes, operation) } #[cfg(test)] @@ -44,6 +53,7 @@ mod tests { use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer::wrapper_allocated_error_message_pointer; use super::read_and_free_cpp_string; + use crate::cpp_string_error::CppStringError; use crate::ffi_contract_error::FfiContractError; #[test] @@ -79,9 +89,30 @@ mod tests { assert_eq!( result, - Err(FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "reported a thrown C++ exception without an error message", + })) + ); + } + + #[test] + fn a_string_that_is_not_utf8_keeps_its_bytes() { + let not_utf8 = c"llama\xff"; + let result = unsafe { + read_and_free_cpp_string( + llama_cpp_bindings_sys::llama_rs_string_dup(not_utf8.as_ptr()), + "llama_rs_json_schema_to_grammar", + "reported a thrown C++ exception without an error message", + ) + }; + + assert_eq!( + result, + Err(CppStringError::NotUtf8 { + operation: "llama_rs_json_schema_to_grammar", + bytes: not_utf8.to_bytes().to_vec(), + source: not_utf8.to_str().expect_err("0xff is not UTF-8"), }) ); } diff --git a/llama-cpp-gbnf/src/gbnf_validation_error.rs b/llama-cpp-gbnf/src/gbnf_validation_error.rs index 2b4414904..5f95531bb 100644 --- a/llama-cpp-gbnf/src/gbnf_validation_error.rs +++ b/llama-cpp-gbnf/src/gbnf_validation_error.rs @@ -1,10 +1,13 @@ use std::ffi::NulError; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; #[derive(Debug, thiserror::Error, PartialEq, Eq)] pub enum GbnfValidationError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] FfiStatus(#[from] FfiStatusError), #[error(transparent)] diff --git a/llama-cpp-gbnf/src/validate_gbnf.rs b/llama-cpp-gbnf/src/validate_gbnf.rs index ca48794e3..c15c02e49 100644 --- a/llama-cpp-gbnf/src/validate_gbnf.rs +++ b/llama-cpp-gbnf/src/validate_gbnf.rs @@ -104,6 +104,7 @@ pub fn validate_gbnf( #[cfg(test)] mod tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ffi::CString; use std::fmt::Write; use std::ptr; @@ -204,10 +205,12 @@ mod tests { "root", ptr::null_mut(), ), - Err(GbnfValidationError::FfiContract(FfiContractError { - operation: "llama_rs_validate_gbnf", - detail: "reported a thrown C++ exception without an error message", - })) + Err(GbnfValidationError::CppString(CppStringError::Missing( + FfiContractError { + operation: "llama_rs_validate_gbnf", + detail: "reported a thrown C++ exception without an error message", + } + ))) ); } From 3d7b1b0f4230183665db1720ff75ff9bce2598a1 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 08:01:22 +0100 Subject: [PATCH 54/82] Report an llguidance step that fails without stopping the parser as an error --- .../src/error/grammar_runtime_error.rs | 2 ++ llama-cpp-bindings/src/grammar_matcher.rs | 19 ++++++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/llama-cpp-bindings/src/error/grammar_runtime_error.rs b/llama-cpp-bindings/src/error/grammar_runtime_error.rs index 9283cbc6b..068ab90da 100644 --- a/llama-cpp-bindings/src/error/grammar_runtime_error.rs +++ b/llama-cpp-bindings/src/error/grammar_runtime_error.rs @@ -1,5 +1,7 @@ #[derive(Debug, PartialEq, Eq, thiserror::Error)] pub enum GrammarRuntimeError { + #[error("the grammar parser failed a step without stopping: {message}")] + FailedWithoutStopping { message: String }, #[error("the grammar parser reached an internal error state: {message}")] InternalParserError { message: String }, #[error("the grammar lexer became too complex: {message}")] diff --git a/llama-cpp-bindings/src/grammar_matcher.rs b/llama-cpp-bindings/src/grammar_matcher.rs index 32c32495f..d52a86d09 100644 --- a/llama-cpp-bindings/src/grammar_matcher.rs +++ b/llama-cpp-bindings/src/grammar_matcher.rs @@ -37,10 +37,10 @@ fn stop_reason_to_result( detail: String, ) -> Result<(), GrammarRuntimeError> { match stop_reason { - StopReason::NotStopped - | StopReason::NoExtension - | StopReason::NoExtensionBias - | StopReason::EndOfSentence => Ok(()), + StopReason::NoExtension | StopReason::NoExtensionBias | StopReason::EndOfSentence => Ok(()), + StopReason::NotStopped => { + Err(GrammarRuntimeError::FailedWithoutStopping { message: detail }) + } StopReason::InternalError => { Err(GrammarRuntimeError::InternalParserError { message: detail }) } @@ -209,7 +209,6 @@ mod tests { #[test] fn benign_stop_reasons_are_ok() { for reason in [ - StopReason::NotStopped, StopReason::NoExtension, StopReason::NoExtensionBias, StopReason::EndOfSentence, @@ -218,6 +217,16 @@ mod tests { } } + #[test] + fn a_failed_step_that_did_not_stop_the_parser_is_an_error() { + assert_eq!( + stop_reason_to_result(StopReason::NotStopped, "boom".to_string()), + Err(GrammarRuntimeError::FailedWithoutStopping { + message: "boom".to_string() + }) + ); + } + #[test] fn internal_error_maps_to_internal_parser_error_with_message() { assert_eq!( From 1bcb0623db8cdcfa4c0ae9b1a23be2d1c7e03bca Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 08:01:50 +0100 Subject: [PATCH 55/82] Keep why tool call arguments are not JSON alongside their text --- .../src/invalid_arguments_json.rs | 21 +++++++++ llama-cpp-bindings-types/src/lib.rs | 1 + .../src/tool_call_arguments.rs | 47 +++++++++++++++---- 3 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 llama-cpp-bindings-types/src/invalid_arguments_json.rs diff --git a/llama-cpp-bindings-types/src/invalid_arguments_json.rs b/llama-cpp-bindings-types/src/invalid_arguments_json.rs new file mode 100644 index 000000000..2e93fab55 --- /dev/null +++ b/llama-cpp-bindings-types/src/invalid_arguments_json.rs @@ -0,0 +1,21 @@ +use serde::Deserialize; +use serde::Serialize; + +/// Why tool call arguments are not JSON: the parser's message and the 1-based line and column +/// where it stopped. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct InvalidArgumentsJson { + pub message: String, + pub line: usize, + pub column: usize, +} + +impl From for InvalidArgumentsJson { + fn from(error: serde_json::Error) -> Self { + Self { + message: error.to_string(), + line: error.line(), + column: error.column(), + } + } +} diff --git a/llama-cpp-bindings-types/src/lib.rs b/llama-cpp-bindings-types/src/lib.rs index 256b5a0ac..f6354c6ac 100644 --- a/llama-cpp-bindings-types/src/lib.rs +++ b/llama-cpp-bindings-types/src/lib.rs @@ -3,6 +3,7 @@ deny(clippy::unwrap_used, clippy::expect_used, clippy::panic) )] +pub mod invalid_arguments_json; pub mod model_provided_tool_call_id; pub mod model_provided_tool_call_id_error; pub mod parsed_chat_message; diff --git a/llama-cpp-bindings-types/src/tool_call_arguments.rs b/llama-cpp-bindings-types/src/tool_call_arguments.rs index 587ec130e..b00ffaf92 100644 --- a/llama-cpp-bindings-types/src/tool_call_arguments.rs +++ b/llama-cpp-bindings-types/src/tool_call_arguments.rs @@ -2,16 +2,27 @@ use serde::Deserialize; use serde::Serialize; use serde_json::Value; +use crate::invalid_arguments_json::InvalidArgumentsJson; + #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub enum ToolCallArguments { ValidJson(Value), - InvalidJson(String), + InvalidJson { + raw: String, + error: InvalidArgumentsJson, + }, } impl ToolCallArguments { #[must_use] pub fn from_string(raw: String) -> Self { - serde_json::from_str::(&raw).map_or_else(|_| Self::InvalidJson(raw), Self::ValidJson) + match serde_json::from_str::(&raw) { + Ok(value) => Self::ValidJson(value), + Err(error) => Self::InvalidJson { + raw, + error: error.into(), + }, + } } } @@ -20,6 +31,7 @@ mod tests { use serde_json::json; use super::ToolCallArguments; + use crate::invalid_arguments_json::InvalidArgumentsJson; #[test] fn from_string_object_returns_valid() { @@ -46,17 +58,34 @@ mod tests { } #[test] - fn from_string_unparseable_returns_invalid() { + fn unparseable_arguments_keep_their_text_and_where_parsing_failed() { let raw = "{not really json".to_owned(); - let result = ToolCallArguments::from_string(raw.clone()); - assert_eq!(result, ToolCallArguments::InvalidJson(raw)); + assert_eq!( + ToolCallArguments::from_string(raw.clone()), + ToolCallArguments::InvalidJson { + raw, + error: InvalidArgumentsJson { + message: "key must be a string at line 1 column 2".to_owned(), + line: 1, + column: 2, + }, + } + ); } #[test] - fn from_string_empty_returns_invalid() { - let result = ToolCallArguments::from_string(String::new()); - - assert_eq!(result, ToolCallArguments::InvalidJson(String::new())); + fn empty_arguments_are_invalid_at_their_end() { + assert_eq!( + ToolCallArguments::from_string(String::new()), + ToolCallArguments::InvalidJson { + raw: String::new(), + error: InvalidArgumentsJson { + message: "EOF while parsing a value at line 1 column 0".to_owned(), + line: 1, + column: 0, + }, + } + ); } } From b1c2517b8578b5198111e0079597de6e8d78a036 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 08:02:19 +0100 Subject: [PATCH 56/82] Drop session load capacity checks llama.cpp already enforces and the unsafe length setting they guarded --- .../src/context/load_seq_state_error.rs | 3 - .../src/context/load_session_error.rs | 3 - llama-cpp-bindings/src/context/session.rs | 58 ++++--------------- 3 files changed, 10 insertions(+), 54 deletions(-) diff --git a/llama-cpp-bindings/src/context/load_seq_state_error.rs b/llama-cpp-bindings/src/context/load_seq_state_error.rs index b3e27983b..a25439e65 100644 --- a/llama-cpp-bindings/src/context/load_seq_state_error.rs +++ b/llama-cpp-bindings/src/context/load_seq_state_error.rs @@ -11,7 +11,4 @@ pub enum LoadSeqStateError { #[error("failed to convert path {0} to str")] PathToStrError(PathBuf), - - #[error("max_length is not large enough to hold {n_out} (was {max_tokens})")] - InsufficientMaxLength { n_out: usize, max_tokens: usize }, } diff --git a/llama-cpp-bindings/src/context/load_session_error.rs b/llama-cpp-bindings/src/context/load_session_error.rs index be514e138..7c2962569 100644 --- a/llama-cpp-bindings/src/context/load_session_error.rs +++ b/llama-cpp-bindings/src/context/load_session_error.rs @@ -11,7 +11,4 @@ pub enum LoadSessionError { #[error("failed to convert path {0} to str")] PathToStrError(PathBuf), - - #[error("max_length is not large enough to hold {n_out} (was {max_tokens})")] - InsufficientMaxLength { n_out: usize, max_tokens: usize }, } diff --git a/llama-cpp-bindings/src/context/session.rs b/llama-cpp-bindings/src/context/session.rs index ab5037c7f..836eec1a0 100644 --- a/llama-cpp-bindings/src/context/session.rs +++ b/llama-cpp-bindings/src/context/session.rs @@ -91,18 +91,13 @@ fn state_data_restore_status_to_result( fn process_session_load_result( success: bool, n_out: usize, - max_tokens: usize, mut tokens: Vec, ) -> Result, LoadSessionError> { if !success { return Err(LoadSessionError::FailedToLoad); } - if n_out > max_tokens { - return Err(LoadSessionError::InsufficientMaxLength { n_out, max_tokens }); - } - - unsafe { tokens.set_len(n_out) }; + tokens.truncate(n_out); Ok(tokens) } @@ -110,18 +105,13 @@ fn process_session_load_result( fn process_seq_load_result( bytes_read: usize, n_out: usize, - max_tokens: usize, mut tokens: Vec, ) -> Result { if bytes_read == 0 { return Err(LoadSeqStateError::FailedToLoad); } - if n_out > max_tokens { - return Err(LoadSeqStateError::InsufficientMaxLength { n_out, max_tokens }); - } - - unsafe { tokens.set_len(n_out) }; + tokens.truncate(n_out); Ok(LoadedSeqState { tokens, bytes_read }) } @@ -172,7 +162,7 @@ impl LlamaContext<'_> { .ok_or_else(|| LoadSessionError::PathToStrError(path.to_path_buf()))?; let cstr = CString::new(path)?; - let mut tokens: Vec = Vec::with_capacity(max_tokens); + let mut tokens = vec![LlamaToken::new(0); max_tokens]; let mut n_out = 0; let tokens_out = tokens @@ -188,7 +178,7 @@ impl LlamaContext<'_> { &raw mut n_out, ) }; - process_session_load_result(success, n_out, max_tokens, tokens) + process_session_load_result(success, n_out, tokens) } /// # Errors @@ -243,7 +233,7 @@ impl LlamaContext<'_> { .ok_or_else(|| LoadSeqStateError::PathToStrError(path.to_path_buf()))?; let cstr = CString::new(path)?; - let mut tokens: Vec = Vec::with_capacity(max_tokens); + let mut tokens = vec![LlamaToken::new(0); max_tokens]; let mut n_out = 0; let tokens_out = tokens @@ -261,7 +251,7 @@ impl LlamaContext<'_> { ) }; - process_seq_load_result(bytes_read, n_out, max_tokens, tokens) + process_seq_load_result(bytes_read, n_out, tokens) } #[must_use] @@ -414,7 +404,7 @@ mod process_session_load_result_tests { let tokens: Vec = (0..100).map(LlamaToken::new).collect(); assert_eq!( - process_session_load_result(true, 10, 100, tokens), + process_session_load_result(true, 10, tokens), Ok((0..10).map(LlamaToken::new).collect()) ); } @@ -422,24 +412,10 @@ mod process_session_load_result_tests { #[test] fn session_load_fails_when_not_successful() { let tokens = vec![LlamaToken::new(0); 100]; - let result = process_session_load_result(false, 0, 100, tokens); + let result = process_session_load_result(false, 0, tokens); assert_eq!(result, Err(LoadSessionError::FailedToLoad)); } - - #[test] - fn session_load_fails_when_n_out_exceeds_max() { - let tokens = vec![LlamaToken::new(0); 100]; - let result = process_session_load_result(true, 101, 100, tokens); - - assert_eq!( - result, - Err(LoadSessionError::InsufficientMaxLength { - n_out: 101, - max_tokens: 100, - }) - ); - } } #[cfg(test)] @@ -454,7 +430,7 @@ mod process_seq_load_result_tests { let tokens: Vec = (0..100).map(LlamaToken::new).collect(); assert_eq!( - process_seq_load_result(42, 10, 100, tokens), + process_seq_load_result(42, 10, tokens), Ok(LoadedSeqState { tokens: (0..10).map(LlamaToken::new).collect(), bytes_read: 42, @@ -465,24 +441,10 @@ mod process_seq_load_result_tests { #[test] fn seq_load_fails_when_zero_bytes_read() { let tokens = vec![LlamaToken::new(0); 100]; - let result = process_seq_load_result(0, 0, 100, tokens); + let result = process_seq_load_result(0, 0, tokens); assert_eq!(result, Err(LoadSeqStateError::FailedToLoad)); } - - #[test] - fn seq_load_fails_when_n_out_exceeds_max() { - let tokens = vec![LlamaToken::new(0); 100]; - let result = process_seq_load_result(42, 101, 100, tokens); - - assert_eq!( - result, - Err(LoadSeqStateError::InsufficientMaxLength { - n_out: 101, - max_tokens: 100, - }) - ); - } } #[cfg(test)] From 78263ca99aa12a9af3eee37faac96f32847a5294 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 08:07:38 +0100 Subject: [PATCH 57/82] Assert the exact grammar-constrained answers instead of any yes or no prefix --- llama-cpp-bindings-tests/src/lib.rs | 1 - .../src/yes_or_no_prefixes.rs | 1 - .../tests/grammar_sampler.rs | 21 +++++++------------ .../tests/llguidance_sampler.rs | 10 ++++----- 4 files changed, 11 insertions(+), 22 deletions(-) delete mode 100644 llama-cpp-bindings-tests/src/yes_or_no_prefixes.rs diff --git a/llama-cpp-bindings-tests/src/lib.rs b/llama-cpp-bindings-tests/src/lib.rs index db5eb248d..5d3e39782 100644 --- a/llama-cpp-bindings-tests/src/lib.rs +++ b/llama-cpp-bindings-tests/src/lib.rs @@ -25,4 +25,3 @@ pub mod tokenize_prompt_with_synthetic_image; pub mod two_sequence_context; pub mod user_turn_chat_session; pub mod weather_chat_session; -pub mod yes_or_no_prefixes; diff --git a/llama-cpp-bindings-tests/src/yes_or_no_prefixes.rs b/llama-cpp-bindings-tests/src/yes_or_no_prefixes.rs deleted file mode 100644 index 9447d9de6..000000000 --- a/llama-cpp-bindings-tests/src/yes_or_no_prefixes.rs +++ /dev/null @@ -1 +0,0 @@ -pub const YES_OR_NO_PREFIXES: [&str; 5] = ["y", "ye", "yes", "n", "no"]; diff --git a/llama-cpp-bindings-tests/tests/grammar_sampler.rs b/llama-cpp-bindings-tests/tests/grammar_sampler.rs index d5ec411f8..b2f238259 100644 --- a/llama-cpp-bindings-tests/tests/grammar_sampler.rs +++ b/llama-cpp-bindings-tests/tests/grammar_sampler.rs @@ -15,12 +15,13 @@ use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; use llama_cpp_bindings_tests::logits_left_by::logits_left_by; use llama_cpp_bindings_tests::single_token::single_token; -use llama_cpp_bindings_tests::yes_or_no_prefixes::YES_OR_NO_PREFIXES; use llama_cpp_fault_injection::armed_fault::ArmedFault; use llama_cpp_gbnf::gbnf_validation_error::GbnfValidationError; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; +const YES_TOKEN_AND_END_OF_GENERATION: u64 = 2; + #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -53,7 +54,7 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 512, n_ubatch = 128, )] -fn grammar_sampler_constrains_the_first_token_to_a_prefix_of_yes_or_no( +fn grammar_sampler_constrains_the_first_token_of_the_answer_to_yes( fixture: &LlamaFixture<'_>, ) -> Result<()> { let model = fixture.model; @@ -97,10 +98,7 @@ fn grammar_sampler_constrains_the_first_token_to_a_prefix_of_yes_or_no( let piece = outcome.piece.raw(); - assert!( - YES_OR_NO_PREFIXES.contains(&piece.to_lowercase().as_str()), - "the grammar must constrain the first token to a prefix of yes or no, got: '{piece}'" - ); + assert_eq!(piece, "Yes"); Ok(()) } @@ -137,7 +135,7 @@ fn grammar_sampler_constrains_the_first_token_to_a_prefix_of_yes_or_no( n_batch = 512, n_ubatch = 128, )] -fn grammar_constrained_chat_answer_is_yes_or_no_content_that_ends_generation( +fn grammar_constrained_chat_answer_is_yes_content_that_ends_generation( fixture: &LlamaFixture<'_>, ) -> Result<()> { let model = fixture.model; @@ -174,12 +172,7 @@ fn grammar_constrained_chat_answer_is_yes_or_no_content_that_ends_generation( } .run(); - let lowercase = outcome.generated.text().to_lowercase(); - assert!( - lowercase == "yes" || lowercase == "no", - "Grammar loop should produce 'yes' or 'no', got: '{}'", - outcome.generated.text() - ); + assert_eq!(outcome.generated.text(), "yes"); assert!( outcome.eog_seen, "loop must terminate via EOG once grammar accepts, not by exhausting the budget; outcome={outcome:?}" @@ -187,7 +180,7 @@ fn grammar_constrained_chat_answer_is_yes_or_no_content_that_ends_generation( assert_eq!(outcome.observed_reasoning, 0); assert_eq!(outcome.observed_undeterminable, 0); assert_eq!(outcome.observed_tool_call, 0); - assert!(outcome.observed_content > 0); + assert_eq!(outcome.observed_content, YES_TOKEN_AND_END_OF_GENERATION); let usage = classifier.into_usage(); assert_eq!(usage.completion_tokens(), outcome.observed_content); diff --git a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs index f7903f51a..980ea6ec7 100644 --- a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs +++ b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs @@ -22,7 +22,6 @@ use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings::token::LlamaToken; use llama_cpp_bindings_tests::logits_left_by::logits_left_by; use llama_cpp_bindings_tests::single_token::single_token; -use llama_cpp_bindings_tests::yes_or_no_prefixes::YES_OR_NO_PREFIXES; use llama_cpp_fault_injection::armed_fault::ArmedFault; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -447,7 +446,9 @@ fn a_sampler_cloned_through_ffi_keeps_its_own_grammar_state( n_batch = 512, n_ubatch = 128, )] -fn llguidance_chain_samples_a_prefix_of_yes_or_no(fixture: &LlamaFixture<'_>) -> Result<()> { +fn llguidance_chain_samples_yes_as_the_first_answer_token( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let model = fixture.model; let backend = fixture.backend; let mut context = LlamaContext::from_model( @@ -473,10 +474,7 @@ fn llguidance_chain_samples_a_prefix_of_yes_or_no(fixture: &LlamaFixture<'_>) -> None, )?; - assert!( - YES_OR_NO_PREFIXES.contains(&piece.as_str()), - "the grammar must constrain the sampled token to a prefix of yes or no, got: '{piece}'" - ); + assert_eq!(piece, "yes"); Ok(()) } From 59c7f290ee20bb6587257fcf3056821e5a9ceff6 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 08:09:11 +0100 Subject: [PATCH 58/82] Decode reasoning classification tests greedily instead of with a tuned seeded sampler --- llama-cpp-bindings-tests/src/lib.rs | 1 - .../src/penalized_nucleus_sampler.rs | 24 ------------------- .../tests/reasoning_classification.rs | 21 ++++++++-------- 3 files changed, 11 insertions(+), 35 deletions(-) delete mode 100644 llama-cpp-bindings-tests/src/penalized_nucleus_sampler.rs diff --git a/llama-cpp-bindings-tests/src/lib.rs b/llama-cpp-bindings-tests/src/lib.rs index 5d3e39782..d266a2076 100644 --- a/llama-cpp-bindings-tests/src/lib.rs +++ b/llama-cpp-bindings-tests/src/lib.rs @@ -14,7 +14,6 @@ pub mod logits_left_by; pub mod media_question_chat_session; pub mod normalize; pub mod paris_weather_tool_call; -pub mod penalized_nucleus_sampler; pub mod prime_memory; pub mod prime_memory_with; pub mod prompt_question; diff --git a/llama-cpp-bindings-tests/src/penalized_nucleus_sampler.rs b/llama-cpp-bindings-tests/src/penalized_nucleus_sampler.rs deleted file mode 100644 index 4bd75ec9e..000000000 --- a/llama-cpp-bindings-tests/src/penalized_nucleus_sampler.rs +++ /dev/null @@ -1,24 +0,0 @@ -use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::sampling::LlamaSampler; - -const SAMPLING_SEED: u32 = 0x00C0_FFEE; - -/// A seeded sampler that penalizes repetition and samples from the nucleus, so a small model -/// does not loop while its answer stays reproducible. -/// -/// # Panics -/// -/// Panics when llama.cpp cannot build one of the samplers. -#[must_use] -pub fn penalized_nucleus_sampler(model: &LlamaModel) -> LlamaSampler { - LlamaSampler::chain([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0) - .expect("the penalties sampler must build"), - LlamaSampler::top_k(40).expect("the top-k sampler must build"), - LlamaSampler::top_p(0.9, 1).expect("the top-p sampler must build"), - LlamaSampler::min_p(0.05, 1).expect("the min-p sampler must build"), - LlamaSampler::temp(0.7).expect("the temperature sampler must build"), - LlamaSampler::dist(SAMPLING_SEED).expect("the distribution sampler must build"), - ]) - .expect("the sampler chain must build") -} diff --git a/llama-cpp-bindings-tests/tests/reasoning_classification.rs b/llama-cpp-bindings-tests/tests/reasoning_classification.rs index 21bfe1b64..596ab7ad5 100644 --- a/llama-cpp-bindings-tests/tests/reasoning_classification.rs +++ b/llama-cpp-bindings-tests/tests/reasoning_classification.rs @@ -8,7 +8,6 @@ use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::chat_answer_generation::ChatAnswerGeneration; use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; -use llama_cpp_bindings_tests::penalized_nucleus_sampler::penalized_nucleus_sampler; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -28,17 +27,19 @@ const MAX_REASONING_ANSWER_TOKENS: i32 = 512; )] fn deepseek_r1_8b_answers_without_reasoning_after_its_reasoning_is_closed( fixture: &LlamaFixture<'_>, -) { +) -> Result<()> { let session = closed_reasoning_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]"); let answer = ChatAnswerGeneration { fixture, session: &session, - sampler: &mut penalized_nucleus_sampler(fixture.model), + sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_ANSWER_TOKENS, } .run(); answer.assert_content_without_reasoning(&session); + + Ok(()) } #[llama_test( @@ -56,7 +57,7 @@ fn deepseek_r1_8b_reasoning_streams_match_the_parsed_answer_when_thinking_is_ena let answer = ChatAnswerGeneration { fixture, session: &session, - sampler: &mut penalized_nucleus_sampler(fixture.model), + sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, } .run(); @@ -147,7 +148,7 @@ fn glm47_answers_without_reasoning_when_thinking_is_disabled( let answer = ChatAnswerGeneration { fixture, session: &session, - sampler: &mut penalized_nucleus_sampler(fixture.model), + sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_ANSWER_TOKENS, } .run(); @@ -172,7 +173,7 @@ fn glm47_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( let answer = ChatAnswerGeneration { fixture, session: &session, - sampler: &mut penalized_nucleus_sampler(fixture.model), + sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, } .run(); @@ -247,7 +248,7 @@ fn qwen35_answers_without_reasoning_when_thinking_is_disabled( let answer = ChatAnswerGeneration { fixture, session: &session, - sampler: &mut penalized_nucleus_sampler(fixture.model), + sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_ANSWER_TOKENS, } .run(); @@ -272,7 +273,7 @@ fn qwen35_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( let answer = ChatAnswerGeneration { fixture, session: &session, - sampler: &mut penalized_nucleus_sampler(fixture.model), + sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, } .run(); @@ -297,7 +298,7 @@ fn qwen36_answers_without_reasoning_when_thinking_is_disabled( let answer = ChatAnswerGeneration { fixture, session: &session, - sampler: &mut penalized_nucleus_sampler(fixture.model), + sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_ANSWER_TOKENS, } .run(); @@ -322,7 +323,7 @@ fn qwen36_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( let answer = ChatAnswerGeneration { fixture, session: &session, - sampler: &mut penalized_nucleus_sampler(fixture.model), + sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, } .run(); From f7fbef2c30da3a122c737354d6aa46c0c2fbe399 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 08:16:21 +0100 Subject: [PATCH 59/82] Assert the exact schema-constrained JSON each model produces and that generation ends --- .../json_schema_constrained_generation.rs | 161 ++++++++++-------- 1 file changed, 91 insertions(+), 70 deletions(-) diff --git a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs index c6196359b..ce6f06661 100644 --- a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs +++ b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs @@ -1,6 +1,6 @@ -use anyhow::Context as _; use anyhow::Result; use serde_json::Value; +use serde_json::json; use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; @@ -12,46 +12,13 @@ use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; +use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; const MAX_ANSWER_TOKENS: i32 = 64; -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn json_schema_grammar_sampler_constrains_the_answer_to_the_schema( - fixture: &LlamaFixture<'_>, -) -> Result<()> { +fn json_schema_constrained_answer(fixture: &LlamaFixture<'_>) -> Result { let model = fixture.model; let mut context = LlamaContext::from_model( model, @@ -59,8 +26,9 @@ fn json_schema_grammar_sampler_constrains_the_answer_to_the_schema( (*fixture.context_params).into_llama_context_params(), )?; - let prompt = "<|im_start|>user\nWhat is 2+2? Respond with a JSON object.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; - let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; + let session = + closed_reasoning_chat_session(model, "What is 2+2? Respond with a JSON object.", "[]"); + let tokens = session.tokenize_prompt()?; let mut batch = LlamaBatch::new(ContextLimits::of(&context)); let mut classifier = SampledTokenClassifier::undeterminable(model); @@ -94,13 +62,73 @@ fn json_schema_grammar_sampler_constrains_the_answer_to_the_schema( "the answer must end once the schema is satisfied; outcome={outcome:?}" ); - let answer = outcome.generated.text().parse::()?; - let answer_object = answer - .as_object() - .context("the schema requires the answer to be a JSON object")?; + Ok(outcome.generated.text().parse::()?) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn deepseek_r1_8b_answers_the_json_schema_with_four(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_eq!( + json_schema_constrained_answer(fixture)?, + json!({"answer": "4"}) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn glm47_flash_answers_the_json_schema_with_a_sentence(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_eq!( + json_schema_constrained_answer(fixture)?, + json!({"answer": "The sum of 2 and 2 is 4."}) + ); + + Ok(()) +} - assert_eq!(answer_object.keys().collect::>(), ["answer"]); - assert!(answer_object["answer"].is_string(), "{answer}"); +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn qwen35_answers_the_json_schema_with_four(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_eq!( + json_schema_constrained_answer(fixture)?, + json!({"answer": "4"}) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn qwen36_answers_the_json_schema_with_an_empty_string(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_eq!( + json_schema_constrained_answer(fixture)?, + json!({"answer": ""}) + ); Ok(()) } @@ -173,39 +201,32 @@ fn an_llguidance_json_schema_constrains_the_answer_to_the_schema( "city": { "type": "string" }, "temperature": { "type": "number" } }, - "required": ["city", "temperature"] + "required": ["city", "temperature"], + "additionalProperties": false }"#; let llg_sampler = LlamaSampler::llguidance(model, "json", schema)?; let mut sampler = LlamaSampler::chain([llg_sampler, LlamaSampler::greedy()?])?; - let mut n_cur = batch.n_tokens(); - let mut decoder = encoding_rs::UTF_8.new_decoder(); - let mut generated = String::new(); - - while n_cur <= 128 { - let token = SampledToken::Content(sampler.sample(&ctx, batch.n_tokens() - 1)?); - - if model.is_eog_token(&token) { - break; - } - - let output_string = model.token_to_piece(&token, &mut decoder, true, None)?; - generated.push_str(&output_string); - - batch.clear(); - batch.add(&token, n_cur, &[0], true)?; - n_cur += 1; - ctx.decode(&batch)?; + let initial_position = batch.n_tokens(); + let outcome = ClassifySampleLoop { + classifier: &mut SampledTokenClassifier::undeterminable(model), + sampler: &mut sampler, + context: &mut ctx, + batch: &mut batch, + initial_position, + max_generated_tokens: MAX_ANSWER_TOKENS, } + .run(); - let parsed = serde_json::Deserializer::from_str(&generated) - .into_iter::() - .next() - .context("model produced no JSON value")??; - - assert!(parsed["city"].is_string(), "{parsed}"); - assert!(parsed["temperature"].is_number(), "{parsed}"); + assert!( + outcome.eog_seen, + "the answer must end once the schema is satisfied; outcome={outcome:?}" + ); + assert_eq!( + outcome.generated.text().parse::()?, + json!({"city": "Paris", "temperature": 22}) + ); Ok(()) } From aec66f934893f9bb20acad12d182b58c77d5902b Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 08:18:16 +0100 Subject: [PATCH 60/82] Assert the exact multimodal transcripts and descriptions instead of substrings --- .../tests/multimodal_audio.rs | 45 +++++++++++-------- .../tests/multimodal_fusion.rs | 22 ++------- 2 files changed, 30 insertions(+), 37 deletions(-) diff --git a/llama-cpp-bindings-tests/tests/multimodal_audio.rs b/llama-cpp-bindings-tests/tests/multimodal_audio.rs index 9f76c04d0..bb48f39b2 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_audio.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_audio.rs @@ -128,21 +128,19 @@ fn transcribe_audio(fixture: &LlamaFixture<'_>, audio_file_name: &str) -> Result } .run(); - Ok(outcome.generated.text().to_lowercase()) + Ok(outcome.generated.text().to_owned()) }, ) } -fn assert_audio_transcription_contains( +fn assert_audio_transcription( fixture: &LlamaFixture<'_>, audio_file_name: &str, - expected_word: &str, + expected_transcript: &str, ) -> Result<()> { - let transcript = transcribe_audio(fixture, audio_file_name)?; - - assert!( - transcript.contains(expected_word), - "transcription should echo the spoken word {expected_word:?}; got: {transcript:?}" + assert_eq!( + transcribe_audio(fixture, audio_file_name)?, + expected_transcript ); Ok(()) @@ -163,13 +161,14 @@ fn assert_audio_transcription_contains( "mmproj-ultravox-v0_5-llama-3_2-1b-f16.gguf" ), )] -fn audio_reaches_the_model_and_produces_a_response(fixture: &LlamaFixture<'_>) -> Result<()> { - let response = transcribe_audio(fixture, "quick_brown_fox.wav")?; - - assert!( - !response.is_empty(), - "the audio must reach the decoder and produce output; this model is too small to \ - follow the transcription instruction reliably, so only the pipeline is asserted here" +fn ultravox_transcribes_the_spoken_sentence_after_a_preamble( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + assert_eq!( + transcribe_audio(fixture, "quick_brown_fox.wav")?, + "I can provide a transcript of the audio. However, please note that the audio itself is \ + not provided. Based on the context, I'll attempt to create a verbatim transcript.\n\nThe \ + quick brown fox jumps over the lazy dog." ); Ok(()) @@ -184,8 +183,12 @@ fn audio_reaches_the_model_and_produces_a_response(fixture: &LlamaFixture<'_>) - n_ubatch = 512, mmproj_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "mmproj-F16.gguf"), )] -fn audio_transcribes_spoken_word(fixture: &LlamaFixture<'_>) -> Result<()> { - assert_audio_transcription_contains(fixture, "quick_brown_fox.wav", "fox") +fn gemma4_transcribes_the_quick_brown_fox_sentence(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_audio_transcription( + fixture, + "quick_brown_fox.wav", + "The quick brown fox jumps over the lazy dog.", + ) } #[llama_test( @@ -197,8 +200,12 @@ fn audio_transcribes_spoken_word(fixture: &LlamaFixture<'_>) -> Result<()> { n_ubatch = 512, mmproj_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "mmproj-F16.gguf"), )] -fn audio_transcribes_uncommon_sentence(fixture: &LlamaFixture<'_>) -> Result<()> { - assert_audio_transcription_contains(fixture, "orange_cat.wav", "fence") +fn gemma4_transcribes_the_orange_cat_sentence(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_audio_transcription( + fixture, + "orange_cat.wav", + "The orange cat sat on the wooden fence.", + ) } #[llama_test( diff --git a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs index 4e4a9ebf7..b77a8617b 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs @@ -23,8 +23,6 @@ use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; -const RECOGNIZED_ANIMALS: [&str; 3] = ["llama", "alpaca", "sheep"]; - const MAX_GENERATED_TOKENS: i32 = 512; const DESCRIBE_INSTRUCTION: &str = "Describe the animal shown in the image, then write the exact words spoken in the audio."; @@ -154,22 +152,10 @@ fn an_image_and_audio_prompt_is_answered_with_both_media_counted_as_input( } .run(); - let description = outcome.generated.text().to_lowercase(); - assert!( - !description.is_empty(), - "model should generate a description from combined image and audio input" - ); - assert!( - RECOGNIZED_ANIMALS - .iter() - .any(|animal| description.contains(animal)), - "the description must name one of the animals a vision encoder plausibly reports for \ - llamas.jpg ({RECOGNIZED_ANIMALS:?}), which is what proves the image reached the \ - output; got: {description:?}" - ); - assert!( - description.contains("fence"), - "description should echo the spoken word \"fence\" from the audio; got: {description:?}" + assert!(outcome.eog_seen, "{outcome:?}"); + assert_eq!( + outcome.generated.text(), + "Animal: The image shows several sheep.\n\nAudio: The orange cat sat on the wooden fence." ); Ok(()) From 085eee095f8e236217c18a920bcfc27dc7f641f8 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 08:19:17 +0100 Subject: [PATCH 61/82] Assert exact tokenization and parse results per model family --- .../tests/chat_answer_parsing.rs | 52 ++++++++++++++----- .../tests/tokenization.rs | 47 +++++++++-------- 2 files changed, 64 insertions(+), 35 deletions(-) diff --git a/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs b/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs index b5b440233..ee2cad0c6 100644 --- a/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs +++ b/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs @@ -12,7 +12,6 @@ use llama_cpp_bindings_tests::recognized_answer::recognized_answer; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; use llama_cpp_bindings_tests::weather_chat_session::weather_chat_session; use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; -use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; use llama_cpp_fault_injection::armed_fault::ArmedFault; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -20,6 +19,19 @@ use llama_cpp_test_harness_macros::llama_test; const GENERATED_REASONING: &str = "step one, step two"; const GENERATED_CONTENT: &str = "actual response"; +fn parsed_content_after_closed_reasoning(fixture: &LlamaFixture<'_>) -> Result { + let outcome = closed_reasoning_chat_session(fixture.model, "hello", "[]").parse( + &ChatOutput::from("hello world"), + ChatOutputCompleteness::Finished, + )?; + + let ChatMessageParseOutcome::Recognized(parsed) = outcome else { + bail!("expected Recognized for plain content; got Unrecognized"); + }; + + Ok(parsed) +} + #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -36,6 +48,21 @@ const GENERATED_CONTENT: &str = "actual response"; n_batch = 128, n_ubatch = 64, )] +fn templates_that_keep_prefill_whitespace_parse_it_into_reasoning_and_content( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + assert_eq!( + parsed_content_after_closed_reasoning(fixture)?, + ParsedChatMessage::new( + "\n\nhello world".to_owned(), + "\nThe question is simple, so I answer it directly.\n".to_owned(), + Vec::new() + ) + ); + + Ok(()) +} + #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -52,18 +79,17 @@ const GENERATED_CONTENT: &str = "actual response"; n_batch = 128, n_ubatch = 64, )] -fn parses_pure_content_response(fixture: &LlamaFixture<'_>) -> Result<()> { - let outcome = closed_reasoning_chat_session(fixture.model, "hello", "[]").parse( - &ChatOutput::from("hello world"), - ChatOutputCompleteness::Finished, - )?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!("expected Recognized for plain content; got Unrecognized"); - }; - assert_eq!(parsed.tool_calls, Vec::::new()); - assert!(!parsed.is_empty()); - assert!(parsed.content.contains("hello world")); +fn qwen_templates_parse_trimmed_prefill_into_reasoning_and_content( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + assert_eq!( + parsed_content_after_closed_reasoning(fixture)?, + ParsedChatMessage::new( + "hello world".to_owned(), + "The question is simple, so I answer it directly.\n".to_owned(), + Vec::new() + ) + ); Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/tokenization.rs b/llama-cpp-bindings-tests/tests/tokenization.rs index b637ef4e3..8188de851 100644 --- a/llama-cpp-bindings-tests/tests/tokenization.rs +++ b/llama-cpp-bindings-tests/tests/tokenization.rs @@ -62,14 +62,25 @@ fn the_first_token_of_tokenized_text_decodes_to_its_first_word( n_batch = 128, n_ubatch = 64, )] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] +fn str_to_token_prepends_bos_for_a_model_that_adds_it(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let tokens_without_bos = + model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; + let expected = model + .token_bos() + .into_iter() + .chain(tokens_without_bos) + .collect::>(); + + assert!(model.adds_bos_token()); + assert_eq!( + model.str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?, + expected + ); + + Ok(()) +} + #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -78,24 +89,16 @@ fn the_first_token_of_tokenized_text_decodes_to_its_first_word( n_batch = 128, n_ubatch = 64, )] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn str_to_token_without_bos_yields_no_more_tokens_than_with_bos( +fn str_to_token_adds_nothing_for_a_model_that_adds_no_bos( fixture: &LlamaFixture<'_>, ) -> Result<()> { let model = fixture.model; - let tokens_with_bos = - model.str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let tokens_without_bos = - model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; - assert!(tokens_with_bos.len() >= tokens_without_bos.len()); + assert!(!model.adds_bos_token()); + assert_eq!( + model.str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?, + model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)? + ); Ok(()) } From 8e5437b6700ee305dd9e2e3b11a1dfcd8a78e131 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 08:20:06 +0100 Subject: [PATCH 62/82] Enumerate the vocabulary as tokens with their pieces and test a known token --- .../tests/vocabulary_enumeration.rs | 42 +++++++------------ llama-cpp-bindings/src/model.rs | 28 ++++++------- .../src/model/vocabulary_token.rs | 8 ++++ 3 files changed, 35 insertions(+), 43 deletions(-) create mode 100644 llama-cpp-bindings/src/model/vocabulary_token.rs diff --git a/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs b/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs index 0296eeb20..3918f5287 100644 --- a/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs +++ b/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs @@ -1,24 +1,10 @@ use anyhow::Result; +use llama_cpp_bindings::model::vocabulary_token::VocabularyToken; +use llama_cpp_bindings_tests::single_token::single_token; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -27,19 +13,19 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 128, n_ubatch = 64, )] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn n_vocab_matches_tokens_iterator_count(fixture: &LlamaFixture<'_>) -> Result<()> { +fn the_vocabulary_enumerates_each_token_at_its_id_with_its_piece( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let model = fixture.model; - let n_vocab = model.n_vocab(); - let count = model.tokens(false).count(); + let hello = single_token(model, "hello"); + + assert_eq!( + model.tokens(false).nth(usize::try_from(hello.0)?), + Some(Ok(VocabularyToken { + token: hello, + piece: "hello".to_owned() + })) + ); - assert_eq!(count, usize::try_from(n_vocab)?); Ok(()) } diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index d443a48fb..8ab2a464b 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -15,6 +15,7 @@ pub mod split_mode; pub mod tokenizer_input; pub mod vocab_type; pub mod vocab_type_from_int_error; +pub mod vocabulary_token; use std::cmp; use std::ffi::c_int; @@ -62,6 +63,7 @@ use crate::model::rope_type::RopeType; use crate::model::tokenizer_input::TokenizerInput; use crate::model::vocab_type::VocabType; use crate::model::vocab_type_from_int_error::VocabTypeFromIntError; +use crate::model::vocabulary_token::VocabularyToken; use crate::sampled_token::SampledToken; use crate::token::LlamaToken; @@ -234,24 +236,20 @@ impl LlamaModel { u32::try_from(n_ctx_train) } + /// Every token of the vocabulary, in id order, with the text it decodes to on its own. pub fn tokens( &self, decode_special: bool, - ) -> impl Iterator)> + '_ { - (0..self.n_vocab()) - .map(LlamaToken::new) - .map(move |llama_token| { - let mut decoder = encoding_rs::UTF_8.new_decoder(); - ( - llama_token, - self.token_to_piece( - &SampledToken::Content(llama_token), - &mut decoder, - decode_special, - None, - ), - ) - }) + ) -> impl Iterator> + '_ { + (0..self.n_vocab()).map(LlamaToken::new).map(move |token| { + self.token_to_piece( + &SampledToken::Content(token), + &mut encoding_rs::UTF_8.new_decoder(), + decode_special, + None, + ) + .map(|piece| VocabularyToken { token, piece }) + }) } #[must_use] diff --git a/llama-cpp-bindings/src/model/vocabulary_token.rs b/llama-cpp-bindings/src/model/vocabulary_token.rs new file mode 100644 index 000000000..e81f98df6 --- /dev/null +++ b/llama-cpp-bindings/src/model/vocabulary_token.rs @@ -0,0 +1,8 @@ +use crate::token::LlamaToken; + +/// A token of the model's vocabulary with the text it decodes to. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct VocabularyToken { + pub token: LlamaToken, + pub piece: String, +} From fc9b315effe8a858c966c568730e5cd3581ffceb Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 08:31:49 +0100 Subject: [PATCH 63/82] State the reasoning each generation prompt holds instead of deriving it from the parser under test --- llama-cpp-bindings-tests/src/chat_answer.rs | 13 +++--- llama-cpp-bindings-tests/src/lib.rs | 1 - .../src/prompted_reasoning.rs | 25 ----------- .../tests/chat_answer_parsing.rs | 42 ++++++++++++++----- .../tests/reasoning_classification.rs | 12 +++--- 5 files changed, 45 insertions(+), 48 deletions(-) delete mode 100644 llama-cpp-bindings-tests/src/prompted_reasoning.rs diff --git a/llama-cpp-bindings-tests/src/chat_answer.rs b/llama-cpp-bindings-tests/src/chat_answer.rs index a9af2d715..6cce95f06 100644 --- a/llama-cpp-bindings-tests/src/chat_answer.rs +++ b/llama-cpp-bindings-tests/src/chat_answer.rs @@ -3,7 +3,6 @@ use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings_types::token_usage::TokenUsage; use crate::classify_sample_loop_outcome::ClassifySampleLoopOutcome; -use crate::prompted_reasoning::prompted_reasoning; fn thinking_tags(session: &ChatSession<'_>) -> Vec { let mut tags = session @@ -57,22 +56,26 @@ impl ChatAnswer { } /// Asserts that the answer reasons, and that the reasoning and content streams are exactly - /// what the session's parser extracts, apart from the reasoning text the generation prompt - /// itself holds, which the parser shows in front of the first generated text. + /// what the session's parser extracts, apart from `prompted_reasoning`, the reasoning text + /// the generation prompt itself holds, which the parser shows in front of the first + /// generated text. /// /// # Panics /// /// Panics when the parser fails on or does not recognize the answer, or when the answer /// does not reason, holds a token of no section, leaks a thinking tag, or streams text the /// parser does not extract. - pub fn assert_reasoning_matches_parse(&self, session: &ChatSession<'_>) { + pub fn assert_reasoning_matches_parse( + &self, + session: &ChatSession<'_>, + prompted_reasoning: &str, + ) { let Self { outcome, usage } = self; let parsed = session .parse(&outcome.generated, ChatOutputCompleteness::Finished) .expect("the session's parser must parse the generated answer") .into_recognized() .expect("the session's parser must recognize the generated answer"); - let prompted_reasoning = prompted_reasoning(session); assert!(outcome.observed_reasoning > 0, "{outcome:?}"); assert_eq!(outcome.observed_undeterminable, 0, "{outcome:?}"); diff --git a/llama-cpp-bindings-tests/src/lib.rs b/llama-cpp-bindings-tests/src/lib.rs index d266a2076..f2a086267 100644 --- a/llama-cpp-bindings-tests/src/lib.rs +++ b/llama-cpp-bindings-tests/src/lib.rs @@ -17,7 +17,6 @@ pub mod paris_weather_tool_call; pub mod prime_memory; pub mod prime_memory_with; pub mod prompt_question; -pub mod prompted_reasoning; pub mod recognized_answer; pub mod single_token; pub mod tokenize_prompt_with_synthetic_image; diff --git a/llama-cpp-bindings-tests/src/prompted_reasoning.rs b/llama-cpp-bindings-tests/src/prompted_reasoning.rs deleted file mode 100644 index 1fcac1d80..000000000 --- a/llama-cpp-bindings-tests/src/prompted_reasoning.rs +++ /dev/null @@ -1,25 +0,0 @@ -use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; -use llama_cpp_bindings::chat_session::ChatSession; - -use crate::recognized_answer::recognized_answer; - -const REASONING_PROBE: &str = "x"; - -/// # Panics -/// -/// Panics when the parser fails on or does not recognize the probe, or when the probe continues -/// neither the prompt's reasoning nor its content. -#[must_use] -pub fn prompted_reasoning(session: &ChatSession<'_>) -> String { - let probed = recognized_answer(session, REASONING_PROBE, ChatOutputCompleteness::Partial); - - if probed.content == REASONING_PROBE && probed.reasoning_content.is_empty() { - return String::new(); - } - - probed - .reasoning_content - .strip_suffix(REASONING_PROBE) - .expect("the probe must continue either the prompt's reasoning or its content") - .to_owned() -} diff --git a/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs b/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs index ee2cad0c6..09d8e2656 100644 --- a/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs +++ b/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs @@ -7,7 +7,6 @@ use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; use llama_cpp_bindings::error::parse_chat_message_error::ParseChatMessageError; use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; -use llama_cpp_bindings_tests::prompted_reasoning::prompted_reasoning; use llama_cpp_bindings_tests::recognized_answer::recognized_answer; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; use llama_cpp_bindings_tests::weather_chat_session::weather_chat_session; @@ -94,6 +93,16 @@ fn qwen_templates_parse_trimmed_prefill_into_reasoning_and_content( Ok(()) } +fn reasoning_parsed_before_the_end_tag(fixture: &LlamaFixture<'_>) -> Result { + let session = user_turn_chat_session(fixture.model, "hello", "[]", true)?; + + Ok(recognized_answer( + &session, + &format!("{GENERATED_REASONING}{GENERATED_CONTENT}"), + ChatOutputCompleteness::Finished, + )) +} + #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -102,6 +111,21 @@ fn qwen_templates_parse_trimmed_prefill_into_reasoning_and_content( n_batch = 128, n_ubatch = 64, )] +fn deepseek_r1_parses_the_reasoning_after_the_newline_its_generation_prompt_opens( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + assert_eq!( + reasoning_parsed_before_the_end_tag(fixture)?, + ParsedChatMessage::new( + GENERATED_CONTENT.to_owned(), + format!("\n{GENERATED_REASONING}"), + Vec::new() + ) + ); + + Ok(()) +} + #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -129,18 +153,14 @@ fn qwen_templates_parse_trimmed_prefill_into_reasoning_and_content( fn parses_the_reasoning_before_the_end_tag_apart_from_the_content( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let session = user_turn_chat_session(fixture.model, "hello", "[]", true)?; - let parsed = recognized_answer( - &session, - &format!("{GENERATED_REASONING}{GENERATED_CONTENT}"), - ChatOutputCompleteness::Finished, - ); - assert_eq!( - parsed.reasoning_content, - format!("{}{GENERATED_REASONING}", prompted_reasoning(&session)) + reasoning_parsed_before_the_end_tag(fixture)?, + ParsedChatMessage::new( + GENERATED_CONTENT.to_owned(), + GENERATED_REASONING.to_owned(), + Vec::new() + ) ); - assert_eq!(parsed.content, GENERATED_CONTENT); Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/reasoning_classification.rs b/llama-cpp-bindings-tests/tests/reasoning_classification.rs index 596ab7ad5..f22a2edc8 100644 --- a/llama-cpp-bindings-tests/tests/reasoning_classification.rs +++ b/llama-cpp-bindings-tests/tests/reasoning_classification.rs @@ -62,7 +62,7 @@ fn deepseek_r1_8b_reasoning_streams_match_the_parsed_answer_when_thinking_is_ena } .run(); - answer.assert_reasoning_matches_parse(&session); + answer.assert_reasoning_matches_parse(&session, "\n"); Ok(()) } @@ -128,7 +128,7 @@ fn gemma4_reasoning_streams_match_the_parsed_answer_when_reasoning_is_prefilled( } .run(); - answer.assert_reasoning_matches_parse(&session); + answer.assert_reasoning_matches_parse(&session, "The user wants one word."); Ok(()) } @@ -178,7 +178,7 @@ fn glm47_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( } .run(); - answer.assert_reasoning_matches_parse(&session); + answer.assert_reasoning_matches_parse(&session, ""); Ok(()) } @@ -228,7 +228,7 @@ fn mistral3_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( } .run(); - answer.assert_reasoning_matches_parse(&session); + answer.assert_reasoning_matches_parse(&session, ""); Ok(()) } @@ -278,7 +278,7 @@ fn qwen35_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( } .run(); - answer.assert_reasoning_matches_parse(&session); + answer.assert_reasoning_matches_parse(&session, ""); Ok(()) } @@ -328,7 +328,7 @@ fn qwen36_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( } .run(); - answer.assert_reasoning_matches_parse(&session); + answer.assert_reasoning_matches_parse(&session, ""); Ok(()) } From 0233ed78a7d325077caea239b8251c2196b5e63a Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 08:32:54 +0100 Subject: [PATCH 64/82] Compare exit codes exactly and fail emitter tests that cannot run the emitter --- llama-cpp-test-harness/src/run.rs | 18 ++---- .../tests/emit_compile_commands.rs | 57 +++++++++++-------- 2 files changed, 38 insertions(+), 37 deletions(-) diff --git a/llama-cpp-test-harness/src/run.rs b/llama-cpp-test-harness/src/run.rs index 435e5ce89..57d0db565 100644 --- a/llama-cpp-test-harness/src/run.rs +++ b/llama-cpp-test-harness/src/run.rs @@ -57,35 +57,25 @@ mod tests { } } - fn as_u8(code: ExitCode) -> u8 { - let formatted = format!("{code:?}"); - formatted - .chars() - .filter(char::is_ascii_digit) - .collect::() - .parse::() - .unwrap_or(255) - } - #[test] fn aggregate_exit_code_zero_when_all_pass() { let code = aggregate_exit_code(&[passing_conclusion(), passing_conclusion()]); - assert_eq!(as_u8(code), 0); + assert_eq!(code, ExitCode::from(0)); } #[test] fn aggregate_exit_code_non_zero_when_any_fails() { let code = aggregate_exit_code(&[passing_conclusion(), failing_conclusion()]); - assert_eq!(as_u8(code), 101); + assert_eq!(code, ExitCode::from(101)); } #[test] fn aggregate_exit_code_reports_success_when_no_phase_ran() { let code = aggregate_exit_code(&[]); - assert_eq!(as_u8(code), 0); + assert_eq!(code, ExitCode::from(0)); } #[test] @@ -97,6 +87,6 @@ mod tests { let code = run(); - assert_eq!(as_u8(code), 2); + assert_eq!(code, ExitCode::from(2)); } } diff --git a/llama-cpp-wrapper-sources/tests/emit_compile_commands.rs b/llama-cpp-wrapper-sources/tests/emit_compile_commands.rs index 4c2678ba1..74b0bfe2a 100644 --- a/llama-cpp-wrapper-sources/tests/emit_compile_commands.rs +++ b/llama-cpp-wrapper-sources/tests/emit_compile_commands.rs @@ -9,11 +9,12 @@ use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; const EMITTER: &str = env!("CARGO_BIN_EXE_llama-cpp-wrapper-sources"); const SYS_DIR: &str = "/repo/llama-cpp-bindings-sys"; -fn emitter_succeeded(arguments: &[&str]) -> bool { +fn emitter_exit_code(arguments: &[&str]) -> Option { Command::new(EMITTER) .args(arguments) .status() - .is_ok_and(|status| status.success()) + .expect("the emitter binary must run") + .code() } fn temporary_path(suffix: &str) -> PathBuf { @@ -28,41 +29,51 @@ fn the_binary_emits_one_entry_and_one_response_line_per_wrapper_source() { let database_path = temporary_path("compile_commands.json"); let response_file_path = temporary_path("wrapper_sources.rsp"); - assert!(emitter_succeeded(&[ - SYS_DIR, - &database_path.display().to_string(), - &response_file_path.display().to_string(), - ])); + assert_eq!( + emitter_exit_code(&[ + SYS_DIR, + &database_path.display().to_string(), + &response_file_path.display().to_string(), + ]), + Some(0) + ); - let database = fs::read_to_string(&database_path).unwrap_or_default(); - let entries: Vec = serde_json::from_str(&database).unwrap_or_default(); + let database = fs::read_to_string(&database_path).expect("the database must be written"); + let entries: Vec = + serde_json::from_str(&database).expect("the database must be a JSON array"); assert_eq!(entries.len(), WRAPPER_SOURCES.len()); - let response_file = fs::read_to_string(&response_file_path).unwrap_or_default(); + let response_file = + fs::read_to_string(&response_file_path).expect("the response file must be written"); assert_eq!(response_file.lines().count(), WRAPPER_SOURCES.len()); - assert!(fs::remove_file(&database_path).is_ok()); - assert!(fs::remove_file(&response_file_path).is_ok()); + fs::remove_file(&database_path).expect("the database must be removable"); + fs::remove_file(&response_file_path).expect("the response file must be removable"); } #[test] fn the_binary_fails_when_the_database_cannot_be_written() { - assert!(!emitter_succeeded(&[ - SYS_DIR, - "/nonexistent-directory/compile_commands.json", - "/nonexistent-directory/wrapper_sources.rsp", - ])); + assert_eq!( + emitter_exit_code(&[ + SYS_DIR, + "/nonexistent-directory/compile_commands.json", + "/nonexistent-directory/wrapper_sources.rsp", + ]), + Some(1) + ); } #[test] fn the_binary_requires_every_argument() { let database_path = temporary_path("unwritten.json"); - assert!(!emitter_succeeded(&[])); - assert!(!emitter_succeeded(&[SYS_DIR])); - assert!(!emitter_succeeded(&[ - SYS_DIR, - &database_path.display().to_string() - ])); + assert_eq!( + [ + emitter_exit_code(&[]), + emitter_exit_code(&[SYS_DIR]), + emitter_exit_code(&[SYS_DIR, &database_path.display().to_string()]), + ], + [Some(1), Some(1), Some(1)] + ); } From 08485845d70e209ce576552176c9b50b494162f7 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 08:44:05 +0100 Subject: [PATCH 65/82] Switch every x86 ggml instruction set explicitly, set Metal explicitly, reconfigure CMake on change, and drop the redundant backends dir chain --- .../fixtures/cmake_project/fixture.c | 2 +- .../src/cargo_directive.rs | 11 +- llama-cpp-bindings-build/src/cmake_build.rs | 7 - .../src/cmake_configuration.rs | 60 ++-- llama-cpp-bindings-build/src/cpu_tuning.rs | 295 +++++++++++------- llama-cpp-bindings-build/src/lib.rs | 1 + .../src/link_directives.rs | 16 +- llama-cpp-bindings-build/src/native_build.rs | 3 - .../src/x86_ggml_switch.rs | 46 +++ llama-cpp-bindings/build.rs | 7 - llama-cpp-bindings/src/lib.rs | 4 - llama-cpp-bindings/src/load_backends.rs | 28 -- llama-cpp-bindings/src/model/params.rs | 6 +- 13 files changed, 285 insertions(+), 201 deletions(-) create mode 100644 llama-cpp-bindings-build/src/x86_ggml_switch.rs delete mode 100644 llama-cpp-bindings/build.rs delete mode 100644 llama-cpp-bindings/src/load_backends.rs diff --git a/llama-cpp-bindings-build/fixtures/cmake_project/fixture.c b/llama-cpp-bindings-build/fixtures/cmake_project/fixture.c index abd6f0d07..f53a279e2 100644 --- a/llama-cpp-bindings-build/fixtures/cmake_project/fixture.c +++ b/llama-cpp-bindings-build/fixtures/cmake_project/fixture.c @@ -1,3 +1,3 @@ int llama_cpp_bindings_build_fixture(void) { - return FIXTURE_VALUE; + return 0; } diff --git a/llama-cpp-bindings-build/src/cargo_directive.rs b/llama-cpp-bindings-build/src/cargo_directive.rs index 0a16cb553..901d21e31 100644 --- a/llama-cpp-bindings-build/src/cargo_directive.rs +++ b/llama-cpp-bindings-build/src/cargo_directive.rs @@ -12,7 +12,6 @@ pub enum CargoDirective { LinkSearch(PathBuf), LinkSearchNative(PathBuf), LinkLibrary(NativeLibrary), - Metadata { key: &'static str, value: String }, Debug(String), } @@ -41,7 +40,6 @@ impl Display for CargoDirective { ) } Self::LinkLibrary(library) => write!(formatter, "cargo:rustc-link-lib={library}"), - Self::Metadata { key, value } => write!(formatter, "cargo:{key}={value}"), Self::Debug(message) => write!(formatter, "cargo:warning=[DEBUG] {message}"), } } @@ -55,7 +53,7 @@ mod tests { use crate::link_kind::LinkKind; use crate::native_library::NativeLibrary; - fn every_directive() -> [CargoDirective; 7] { + fn every_directive() -> [CargoDirective; 6] { [ CargoDirective::RerunIfChanged(PathBuf::from("wrapper.h")), CargoDirective::RerunIfEnvChanged("CUDAARCHS".to_owned()), @@ -65,10 +63,6 @@ mod tests { link_kind: LinkKind::Static, name: "llama", }), - CargoDirective::Metadata { - key: "backends_dir", - value: "/out/backends".to_owned(), - }, CargoDirective::Debug("TARGET: x86_64-unknown-linux-gnu".to_owned()), ] } @@ -83,7 +77,6 @@ mod tests { "cargo:rustc-link-search=/out/lib", "cargo:rustc-link-search=native=/out/build/common", "cargo:rustc-link-lib=static=llama", - "cargo:backends_dir=/out/backends", "cargo:warning=[DEBUG] TARGET: x86_64-unknown-linux-gnu", ] ); @@ -93,7 +86,7 @@ mod tests { fn only_debug_messages_are_debug_directives() { assert_eq!( every_directive().map(|directive| directive.is_debug()), - [false, false, false, false, false, false, true] + [false, false, false, false, false, true] ); } } diff --git a/llama-cpp-bindings-build/src/cmake_build.rs b/llama-cpp-bindings-build/src/cmake_build.rs index de4f8b193..805413013 100644 --- a/llama-cpp-bindings-build/src/cmake_build.rs +++ b/llama-cpp-bindings-build/src/cmake_build.rs @@ -15,7 +15,6 @@ pub struct CmakeBuild { pub target_triple: String, pub host_triple: String, pub defines: Vec, - pub compiler_flags: Vec, pub required_directories: Vec, } @@ -42,17 +41,11 @@ impl CmakeBuild { config.define(name, value); } - for compiler_flag in &self.compiler_flags { - config.cflag(compiler_flag); - config.cxxflag(compiler_flag); - } - config .target(&self.target_triple) .host(&self.host_triple) .out_dir(&self.out_dir) .profile(&self.profile) - .always_configure(false) .build(); Ok(()) diff --git a/llama-cpp-bindings-build/src/cmake_configuration.rs b/llama-cpp-bindings-build/src/cmake_configuration.rs index c27d99864..2d1eb7ae4 100644 --- a/llama-cpp-bindings-build/src/cmake_configuration.rs +++ b/llama-cpp-bindings-build/src/cmake_configuration.rs @@ -31,7 +31,6 @@ fn base_defines() -> Vec { #[derive(Clone, Debug, Eq, PartialEq)] pub struct CmakeConfiguration { pub defines: Vec, - pub compiler_flags: Vec, pub required_directories: Vec, pub directives: Vec, } @@ -45,12 +44,10 @@ impl CmakeConfiguration { pub fn of(context: &BuildContext, environment: &BuildEnvironment) -> Result { let CpuTuning { defines: cpu_defines, - compiler_flags, debug_messages, } = CpuTuning::of(context, environment)?; let mut configuration = Self { defines: [base_defines(), cpu_defines].concat(), - compiler_flags, required_directories: Vec::new(), directives: debug_messages, }; @@ -141,6 +138,13 @@ impl CmakeConfiguration { self.defines.push(CmakeDefine::new("GGML_HIP", "ON")); } + if context.target_os == TargetOs::Apple { + self.defines.push(CmakeDefine::switch( + "GGML_METAL", + environment.has(BuildFeature::Metal), + )); + } + Ok(()) } @@ -192,10 +196,6 @@ impl CmakeConfiguration { CmakeDefine::new("GGML_CPU_ALL_VARIANTS", "ON"), CmakeDefine::new("GGML_BACKEND_DIR", backends_dir.clone()), ]); - self.directives.push(CargoDirective::Metadata { - key: "backends_dir", - value: backends_dir.display().to_string(), - }); self.required_directories.push(backends_dir); } } @@ -216,6 +216,7 @@ mod tests { use crate::build_feature::BuildFeature; use crate::cargo_directive::CargoDirective; use crate::cmake_define::CmakeDefine; + use crate::cpu_tuning::CpuTuning; use crate::link_kind::LinkKind; use crate::native_library::NativeLibrary; @@ -232,9 +233,12 @@ mod tests { features: BitFlags, overrides: &[(&str, &str)], ) -> CmakeConfiguration { - let mut configuration = configured(features, overrides); + let environment = cargo_environment(features, overrides); + let context = context_of(&environment); + let cpu_defines = CpuTuning::of(&context, &environment).unwrap().defines; + let mut configuration = CmakeConfiguration::of(&context, &environment).unwrap(); - configuration.defines.drain(..9); + configuration.defines.drain(..9 + cpu_defines.len()); configuration } @@ -266,10 +270,23 @@ mod tests { CmakeDefine::new("MTMD_VIDEO", "OFF"), CmakeDefine::new("LLAMA_OPENSSL", "OFF"), CmakeDefine::new("GGML_NATIVE", "OFF"), + CmakeDefine::new("GGML_SSE42", "OFF"), + CmakeDefine::new("GGML_AVX", "OFF"), + CmakeDefine::new("GGML_AVX_VNNI", "OFF"), + CmakeDefine::new("GGML_AVX2", "OFF"), + CmakeDefine::new("GGML_BMI2", "OFF"), + CmakeDefine::new("GGML_FMA", "OFF"), + CmakeDefine::new("GGML_F16C", "OFF"), + CmakeDefine::new("GGML_AVX512", "OFF"), + CmakeDefine::new("GGML_AVX512_VBMI", "OFF"), + CmakeDefine::new("GGML_AVX512_VNNI", "OFF"), + CmakeDefine::new("GGML_AVX512_BF16", "OFF"), + CmakeDefine::new("GGML_AMX_TILE", "OFF"), + CmakeDefine::new("GGML_AMX_INT8", "OFF"), + CmakeDefine::new("GGML_AMX_BF16", "OFF"), CmakeDefine::new("BUILD_SHARED_LIBS", "OFF"), CmakeDefine::new("GGML_OPENMP", "OFF"), ], - compiler_flags: Vec::new(), required_directories: Vec::new(), directives: vec![compiling_with_no_target_features()], } @@ -285,13 +302,12 @@ mod tests { ), CmakeConfiguration { defines: vec![ - CmakeDefine::new("GGML_NATIVE", "OFF"), CmakeDefine::new("BUILD_SHARED_LIBS", "OFF"), CmakeDefine::new("GGML_BLAS", "OFF"), CmakeDefine::new("GGML_VULKAN", "ON"), + CmakeDefine::new("GGML_METAL", "ON"), CmakeDefine::new("GGML_OPENMP", "OFF"), ], - compiler_flags: Vec::new(), required_directories: Vec::new(), directives: vec![compiling_with_no_target_features()], } @@ -306,7 +322,6 @@ mod tests { let android_defines = |stl: Vec| { [ vec![ - CmakeDefine::new("GGML_NATIVE", "OFF"), CmakeDefine::new("BUILD_SHARED_LIBS", "OFF"), CmakeDefine::new( "CMAKE_TOOLCHAIN_FILE", @@ -325,7 +340,6 @@ mod tests { }; let android_configuration = |stl: Vec| CmakeConfiguration { defines: android_defines(stl), - compiler_flags: Vec::new(), required_directories: Vec::new(), directives: vec![ compiling_with_no_target_features(), @@ -364,7 +378,6 @@ mod tests { let gpu_defines = |cuda_architectures: Vec| { [ vec![ - CmakeDefine::new("GGML_NATIVE", "OFF"), CmakeDefine::new("BUILD_SHARED_LIBS", "OFF"), CmakeDefine::new("GGML_VULKAN", "ON"), CmakeDefine::new("GGML_CUDA", "ON"), @@ -398,7 +411,6 @@ mod tests { [ CmakeConfiguration { defines: gpu_defines(vec![CmakeDefine::new("CMAKE_CUDA_ARCHITECTURES", "89")]), - compiler_flags: Vec::new(), required_directories: Vec::new(), directives: vulkan_directives(vec![CargoDirective::LinkSearch(PathBuf::from( "/vulkan/lib" @@ -406,7 +418,6 @@ mod tests { }, CmakeConfiguration { defines: gpu_defines(Vec::new()), - compiler_flags: Vec::new(), required_directories: Vec::new(), directives: vulkan_directives(Vec::new()), }, @@ -419,7 +430,6 @@ mod tests { assert_eq!( after_the_base_switches(BitFlags::from(BuildFeature::Openmp), &[]).defines, [ - CmakeDefine::new("GGML_NATIVE", "OFF"), CmakeDefine::new("BUILD_SHARED_LIBS", "OFF"), CmakeDefine::new("GGML_OPENMP", "ON"), ] @@ -435,13 +445,11 @@ mod tests { ), CmakeConfiguration { defines: vec![ - CmakeDefine::new("GGML_NATIVE", "OFF"), CmakeDefine::new("BUILD_SHARED_LIBS", "OFF"), CmakeDefine::new("GGML_OPENMP", "OFF"), CmakeDefine::new("LLAMA_USE_SYSTEM_GGML", "ON"), CmakeDefine::new("GGML_DIR", "/ggml/lib/cmake/ggml"), ], - compiler_flags: Vec::new(), required_directories: Vec::new(), directives: vec![ compiling_with_no_target_features(), @@ -460,22 +468,16 @@ mod tests { ), CmakeConfiguration { defines: vec![ - CmakeDefine::new("GGML_NATIVE", "OFF"), CmakeDefine::new("BUILD_SHARED_LIBS", "ON"), CmakeDefine::new("GGML_OPENMP", "OFF"), CmakeDefine::new("GGML_BACKEND_DL", "ON"), CmakeDefine::new("GGML_CPU_ALL_VARIANTS", "ON"), CmakeDefine::new("GGML_BACKEND_DIR", "/out/cmake/backends"), ], - compiler_flags: Vec::new(), required_directories: vec![PathBuf::from("/out/cmake/backends")], - directives: vec![ - compiling_with_no_target_features(), - CargoDirective::Metadata { - key: "backends_dir", - value: "/out/cmake/backends".to_owned(), - }, - ], + directives: vec![CargoDirective::Debug( + "Building every CPU variant for ggml to select at runtime".to_owned() + )], } ); } diff --git a/llama-cpp-bindings-build/src/cpu_tuning.rs b/llama-cpp-bindings-build/src/cpu_tuning.rs index 4a88c6575..86cff30d4 100644 --- a/llama-cpp-bindings-build/src/cpu_tuning.rs +++ b/llama-cpp-bindings-build/src/cpu_tuning.rs @@ -1,9 +1,10 @@ use crate::build_context::BuildContext; use crate::build_environment::BuildEnvironment; use crate::build_error::BuildError; +use crate::build_feature::BuildFeature; use crate::cargo_directive::CargoDirective; use crate::cmake_define::CmakeDefine; -use crate::target_os::TargetOs; +use crate::x86_ggml_switch::X86GgmlSwitch; fn codegen_options(encoded_rustflags: &str) -> Vec<&str> { let mut options = Vec::new(); @@ -31,36 +32,92 @@ fn target_cpu_of(encoded_rustflags: &str) -> Option { .map(ToOwned::to_owned) } -fn map_cpu_feature_to_ggml(feature: &str) -> Option<&'static str> { - match feature { - "avx" => Some("GGML_AVX"), - "avx2" => Some("GGML_AVX2"), - "avx512bf16" => Some("GGML_AVX512_BF16"), - "avx512vbmi" => Some("GGML_AVX512_VBMI"), - "avx512vnni" => Some("GGML_AVX512_VNNI"), - "avxvnni" => Some("GGML_AVX_VNNI"), - "bmi2" => Some("GGML_BMI2"), - "f16c" => Some("GGML_F16C"), - "fma" => Some("GGML_FMA"), - "sse4.2" => Some("GGML_SSE42"), - _ => None, - } -} +const X86_GGML_SWITCHES: [X86GgmlSwitch; 14] = [ + X86GgmlSwitch { + name: "GGML_SSE42", + target_features: &["sse4.2"], + }, + X86GgmlSwitch { + name: "GGML_AVX", + target_features: &["avx"], + }, + X86GgmlSwitch { + name: "GGML_AVX_VNNI", + target_features: &["avxvnni"], + }, + X86GgmlSwitch { + name: "GGML_AVX2", + target_features: &["avx2"], + }, + X86GgmlSwitch { + name: "GGML_BMI2", + target_features: &["bmi2"], + }, + X86GgmlSwitch { + name: "GGML_FMA", + target_features: &["fma"], + }, + X86GgmlSwitch { + name: "GGML_F16C", + target_features: &["f16c"], + }, + X86GgmlSwitch { + name: "GGML_AVX512", + target_features: &["avx512f", "avx512cd", "avx512vl", "avx512dq", "avx512bw"], + }, + X86GgmlSwitch { + name: "GGML_AVX512_VBMI", + target_features: &["avx512vbmi"], + }, + X86GgmlSwitch { + name: "GGML_AVX512_VNNI", + target_features: &["avx512vnni"], + }, + X86GgmlSwitch { + name: "GGML_AVX512_BF16", + target_features: &["avx512bf16"], + }, + X86GgmlSwitch { + name: "GGML_AMX_TILE", + target_features: &["amx-tile"], + }, + X86GgmlSwitch { + name: "GGML_AMX_INT8", + target_features: &["amx-int8"], + }, + X86GgmlSwitch { + name: "GGML_AMX_BF16", + target_features: &["amx-bf16"], + }, +]; /// How ggml is compiled for the CPU rustc compiles for, and the debug messages explaining it. #[derive(Clone, Debug, Eq, PartialEq)] pub struct CpuTuning { pub defines: Vec, - pub compiler_flags: Vec, pub debug_messages: Vec, } impl CpuTuning { + /// ggml builds every CPU variant and selects one at runtime when backends load dynamically. + /// Otherwise `target-cpu=native` lets ggml detect the CPU, and any other build switches + /// every x86 instruction set explicitly on or off, the way rustc's target features describe + /// the CPU, because ggml would enable AVX2-class defaults on its own. + /// /// # Errors /// /// Returns [`BuildError::Environment`] when the rustflags or the target features are not /// Unicode. pub fn of(context: &BuildContext, environment: &BuildEnvironment) -> Result { + if environment.has(BuildFeature::DynamicBackends) { + return Ok(Self { + defines: vec![CmakeDefine::new("GGML_NATIVE", "OFF")], + debug_messages: vec![CargoDirective::Debug( + "Building every CPU variant for ggml to select at runtime".to_owned(), + )], + }); + } + let target_cpu = environment .optional("CARGO_ENCODED_RUSTFLAGS")? .and_then(|rustflags| target_cpu_of(&rustflags)); @@ -68,7 +125,6 @@ impl CpuTuning { if target_cpu.as_deref() == Some("native") { return Ok(Self { defines: vec![CmakeDefine::new("GGML_NATIVE", "ON")], - compiler_flags: Vec::new(), debug_messages: vec![CargoDirective::Debug( "Detected target-cpu=native, compiling with GGML_NATIVE".to_owned(), )], @@ -78,70 +134,26 @@ impl CpuTuning { let target_features = environment .optional("CARGO_CFG_TARGET_FEATURE")? .unwrap_or_default(); - let compiler_flags = target_cpu - .iter() - .map(|cpu| format!("-march={cpu}")) - .collect::>(); + let enabled_features = target_features.split(',').collect::>(); let mut defines = vec![CmakeDefine::new("GGML_NATIVE", "OFF")]; - defines.extend( - target_features - .split(',') - .filter_map(map_cpu_feature_to_ggml) - .map(|ggml_switch| CmakeDefine::new(ggml_switch, "ON")), - ); - - if context.cargo_cfg_target_arch == "aarch64" && context.target_os == TargetOs::Linux { - defines.push(CmakeDefine::new("GGML_CPU_ARM_ARCH", "armv8-a")); + if matches!(context.cargo_cfg_target_arch.as_str(), "x86" | "x86_64") { + defines.extend( + X86_GGML_SWITCHES + .iter() + .map(|switch| switch.define_for(&enabled_features)), + ); } Ok(Self { defines, - debug_messages: compiler_flags - .iter() - .map(|flag| format!("Setting baseline architecture: {flag}")) - .chain([format!("Compiling with target features: {target_features}")]) - .map(CargoDirective::Debug) - .collect(), - compiler_flags, + debug_messages: vec![CargoDirective::Debug(format!( + "Compiling with target features: {target_features}" + ))], }) } } -#[cfg(test)] -mod cpu_feature_mapping_tests { - use super::map_cpu_feature_to_ggml; - - #[test] - fn every_supported_rust_cpu_feature_maps_to_its_ggml_switch() { - assert_eq!(map_cpu_feature_to_ggml("avx"), Some("GGML_AVX")); - assert_eq!(map_cpu_feature_to_ggml("avx2"), Some("GGML_AVX2")); - assert_eq!( - map_cpu_feature_to_ggml("avx512bf16"), - Some("GGML_AVX512_BF16") - ); - assert_eq!( - map_cpu_feature_to_ggml("avx512vbmi"), - Some("GGML_AVX512_VBMI") - ); - assert_eq!( - map_cpu_feature_to_ggml("avx512vnni"), - Some("GGML_AVX512_VNNI") - ); - assert_eq!(map_cpu_feature_to_ggml("avxvnni"), Some("GGML_AVX_VNNI")); - assert_eq!(map_cpu_feature_to_ggml("bmi2"), Some("GGML_BMI2")); - assert_eq!(map_cpu_feature_to_ggml("f16c"), Some("GGML_F16C")); - assert_eq!(map_cpu_feature_to_ggml("fma"), Some("GGML_FMA")); - assert_eq!(map_cpu_feature_to_ggml("sse4.2"), Some("GGML_SSE42")); - } - - #[test] - fn unsupported_rust_cpu_feature_does_not_enable_a_ggml_switch() { - assert_eq!(map_cpu_feature_to_ggml("aes"), None); - assert_eq!(map_cpu_feature_to_ggml(""), None); - } -} - #[cfg(test)] mod target_cpu_of_tests { use super::target_cpu_of; @@ -198,25 +210,50 @@ mod cpu_tuning_tests { use crate::build_context::tests::cargo_environment; use crate::build_context::tests::context_of; use crate::build_environment::tests::with_non_unicode; + use crate::build_feature::BuildFeature; use crate::cargo_directive::CargoDirective; use crate::cmake_define::CmakeDefine; - fn tuned(overrides: &[(&str, &str)]) -> CpuTuning { - let environment = cargo_environment(BitFlags::empty(), overrides); + fn tuned(features: BitFlags, overrides: &[(&str, &str)]) -> CpuTuning { + let environment = cargo_environment(features, overrides); CpuTuning::of(&context_of(&environment), &environment).unwrap() } + fn x86_switches(enabled: &[&'static str]) -> Vec { + [ + "GGML_SSE42", + "GGML_AVX", + "GGML_AVX_VNNI", + "GGML_AVX2", + "GGML_BMI2", + "GGML_FMA", + "GGML_F16C", + "GGML_AVX512", + "GGML_AVX512_VBMI", + "GGML_AVX512_VNNI", + "GGML_AVX512_BF16", + "GGML_AMX_TILE", + "GGML_AMX_INT8", + "GGML_AMX_BF16", + ] + .into_iter() + .map(|name| CmakeDefine::switch(name, enabled.contains(&name))) + .collect() + } + #[test] fn a_native_target_cpu_lets_ggml_detect_the_cpu() { assert_eq!( - tuned(&[ - ("CARGO_ENCODED_RUSTFLAGS", "-Ctarget-cpu=native"), - ("CARGO_CFG_TARGET_FEATURE", "avx2"), - ]), + tuned( + BitFlags::empty(), + &[ + ("CARGO_ENCODED_RUSTFLAGS", "-Ctarget-cpu=native"), + ("CARGO_CFG_TARGET_FEATURE", "avx2"), + ] + ), CpuTuning { defines: vec![CmakeDefine::new("GGML_NATIVE", "ON")], - compiler_flags: Vec::new(), debug_messages: vec![CargoDirective::Debug( "Detected target-cpu=native, compiling with GGML_NATIVE".to_owned() )], @@ -225,44 +262,66 @@ mod cpu_tuning_tests { } #[test] - fn a_named_target_cpu_sets_the_baseline_and_target_features_enable_their_ggml_switches() { + fn a_baseline_x86_build_turns_every_instruction_set_ggml_would_assume_off() { assert_eq!( - tuned(&[ - ("CARGO_ENCODED_RUSTFLAGS", "-Ctarget-cpu=znver4"), - ("CARGO_CFG_TARGET_FEATURE", "aes,avx2,fma"), - ]), + tuned( + BitFlags::empty(), + &[("CARGO_CFG_TARGET_FEATURE", "fxsr,sse,sse2")] + ), CpuTuning { - defines: vec![ - CmakeDefine::new("GGML_NATIVE", "OFF"), - CmakeDefine::new("GGML_AVX2", "ON"), - CmakeDefine::new("GGML_FMA", "ON"), - ], - compiler_flags: vec!["-march=znver4".to_owned()], - debug_messages: vec![ - CargoDirective::Debug( - "Setting baseline architecture: -march=znver4".to_owned() - ), - CargoDirective::Debug( - "Compiling with target features: aes,avx2,fma".to_owned() - ), - ], + defines: [ + vec![CmakeDefine::new("GGML_NATIVE", "OFF")], + x86_switches(&[]), + ] + .concat(), + debug_messages: vec![CargoDirective::Debug( + "Compiling with target features: fxsr,sse,sse2".to_owned() + )], } ); } #[test] - fn aarch64_linux_without_a_target_cpu_builds_for_the_armv8_baseline() { + fn a_named_target_cpu_switches_on_exactly_the_instruction_sets_its_target_features_enable() { + assert_eq!( + tuned( + BitFlags::empty(), + &[ + ("CARGO_ENCODED_RUSTFLAGS", "-Ctarget-cpu=znver4"), + ( + "CARGO_CFG_TARGET_FEATURE", + "avx,avx2,avx512f,avx512cd,avx512vl,avx512dq,bmi2,f16c,fma,sse4.2" + ), + ] + ) + .defines, + [ + vec![CmakeDefine::new("GGML_NATIVE", "OFF")], + x86_switches(&[ + "GGML_SSE42", + "GGML_AVX", + "GGML_AVX2", + "GGML_BMI2", + "GGML_FMA", + "GGML_F16C", + ]), + ] + .concat() + ); + } + + #[test] + fn an_aarch64_build_leaves_the_architecture_to_the_compiler_baseline() { assert_eq!( - tuned(&[ - ("CARGO_CFG_TARGET_ARCH", "aarch64"), - ("TARGET", "aarch64-unknown-linux-gnu"), - ]), + tuned( + BitFlags::empty(), + &[ + ("CARGO_CFG_TARGET_ARCH", "aarch64"), + ("TARGET", "aarch64-unknown-linux-gnu"), + ] + ), CpuTuning { - defines: vec![ - CmakeDefine::new("GGML_NATIVE", "OFF"), - CmakeDefine::new("GGML_CPU_ARM_ARCH", "armv8-a"), - ], - compiler_flags: Vec::new(), + defines: vec![CmakeDefine::new("GGML_NATIVE", "OFF")], debug_messages: vec![CargoDirective::Debug( "Compiling with target features: ".to_owned() )], @@ -270,6 +329,26 @@ mod cpu_tuning_tests { ); } + #[test] + fn dynamic_backends_build_every_cpu_variant_without_tuning() { + assert_eq!( + tuned( + BuildFeature::DynamicBackends | BuildFeature::DynamicLink, + &[ + ("CARGO_CFG_TARGET_ARCH", "aarch64"), + ("TARGET", "aarch64-unknown-linux-gnu"), + ("CARGO_ENCODED_RUSTFLAGS", "-Ctarget-cpu=native"), + ] + ), + CpuTuning { + defines: vec![CmakeDefine::new("GGML_NATIVE", "OFF")], + debug_messages: vec![CargoDirective::Debug( + "Building every CPU variant for ggml to select at runtime".to_owned() + )], + } + ); + } + #[test] fn rustflags_or_target_features_that_are_not_unicode_are_refused() { let context = context_of(&cargo_environment(BitFlags::empty(), &[])); diff --git a/llama-cpp-bindings-build/src/lib.rs b/llama-cpp-bindings-build/src/lib.rs index 64dc80f3b..6e72f0022 100644 --- a/llama-cpp-bindings-build/src/lib.rs +++ b/llama-cpp-bindings-build/src/lib.rs @@ -27,6 +27,7 @@ mod privatized_field; mod rebuild_triggers; mod target_os; mod wrapper_compilation; +mod x86_ggml_switch; use std::io::Write; diff --git a/llama-cpp-bindings-build/src/link_directives.rs b/llama-cpp-bindings-build/src/link_directives.rs index 2b97369fa..c80b79ccd 100644 --- a/llama-cpp-bindings-build/src/link_directives.rs +++ b/llama-cpp-bindings-build/src/link_directives.rs @@ -217,13 +217,17 @@ fn openmp_links(context: &BuildContext, environment: &BuildEnvironment) -> Vec Vec { match context.target_os { TargetOs::Linux => vec![link(LinkKind::Dylib, "stdc++")], - TargetOs::Apple => vec![ + TargetOs::Apple if environment.has(BuildFeature::Metal) => vec![ link(LinkKind::Framework, "Foundation"), link(LinkKind::Framework, "Metal"), link(LinkKind::Framework, "MetalKit"), link(LinkKind::Framework, "Accelerate"), link(LinkKind::Unspecified, "c++"), ], + TargetOs::Apple => vec![ + link(LinkKind::Framework, "Accelerate"), + link(LinkKind::Unspecified, "c++"), + ], TargetOs::Android(_) if environment.has(BuildFeature::StaticStdcxx) => vec![ link(LinkKind::Unspecified, "c++_static"), link(LinkKind::Unspecified, "c++abi"), @@ -582,9 +586,13 @@ mod tests { } #[test] - fn apple_and_android_targets_link_their_platform_cxx_runtime() { + fn apple_and_android_targets_link_their_platform_cxx_runtime_and_metal_only_with_metal() { assert_eq!( [ + platform_libraries( + BitFlags::from(BuildFeature::Metal), + &[("CARGO_CFG_TARGET_OS", "macos")] + ), platform_libraries(BitFlags::empty(), &[("CARGO_CFG_TARGET_OS", "macos")]), platform_libraries( BitFlags::from(BuildFeature::StaticStdcxx), @@ -604,6 +612,10 @@ mod tests { library(LinkKind::Framework, "Accelerate"), library(LinkKind::Unspecified, "c++"), ], + vec![ + library(LinkKind::Framework, "Accelerate"), + library(LinkKind::Unspecified, "c++"), + ], vec![ library(LinkKind::Unspecified, "c++_static"), library(LinkKind::Unspecified, "c++abi"), diff --git a/llama-cpp-bindings-build/src/native_build.rs b/llama-cpp-bindings-build/src/native_build.rs index 25e1dd127..272605168 100644 --- a/llama-cpp-bindings-build/src/native_build.rs +++ b/llama-cpp-bindings-build/src/native_build.rs @@ -29,7 +29,6 @@ impl NativeBuild { pub fn of(context: &BuildContext, environment: &BuildEnvironment) -> Result { let CmakeConfiguration { defines, - compiler_flags, required_directories, directives: cmake_directives, } = CmakeConfiguration::of(context, environment)?; @@ -44,7 +43,6 @@ impl NativeBuild { target_triple: context.target_triple.clone(), host_triple: context.host_triple.clone(), defines, - compiler_flags, required_directories, }, directives: [ @@ -135,7 +133,6 @@ mod tests { target_triple: HOST_TRIPLE.to_owned(), host_triple: HOST_TRIPLE.to_owned(), defines: vec![CmakeDefine::new("FIXTURE_LIBRARY_NAME", "fixture")], - compiler_flags: vec!["-DFIXTURE_VALUE=1".to_owned()], required_directories: vec![root.join("cmake/backends")], }, directives: vec![ diff --git a/llama-cpp-bindings-build/src/x86_ggml_switch.rs b/llama-cpp-bindings-build/src/x86_ggml_switch.rs new file mode 100644 index 000000000..6398e1299 --- /dev/null +++ b/llama-cpp-bindings-build/src/x86_ggml_switch.rs @@ -0,0 +1,46 @@ +use crate::cmake_define::CmakeDefine; + +/// A ggml x86 instruction-set switch and the rustc target features that must all be enabled +/// to turn it on. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct X86GgmlSwitch { + pub name: &'static str, + pub target_features: &'static [&'static str], +} + +impl X86GgmlSwitch { + #[must_use] + pub fn define_for(&self, enabled_features: &[&str]) -> CmakeDefine { + CmakeDefine::switch( + self.name, + self.target_features + .iter() + .all(|target_feature| enabled_features.contains(target_feature)), + ) + } +} + +#[cfg(test)] +mod tests { + use super::X86GgmlSwitch; + use crate::cmake_define::CmakeDefine; + + const AVX512: X86GgmlSwitch = X86GgmlSwitch { + name: "GGML_AVX512", + target_features: &["avx512f", "avx512bw"], + }; + + #[test] + fn a_switch_is_on_only_when_every_target_feature_it_needs_is_enabled() { + assert_eq!( + [ + AVX512.define_for(&["avx512f", "avx512bw", "avx2"]), + AVX512.define_for(&["avx512f", "avx2"]), + ], + [ + CmakeDefine::switch("GGML_AVX512", true), + CmakeDefine::switch("GGML_AVX512", false), + ] + ); + } +} diff --git a/llama-cpp-bindings/build.rs b/llama-cpp-bindings/build.rs deleted file mode 100644 index 5294abc51..000000000 --- a/llama-cpp-bindings/build.rs +++ /dev/null @@ -1,7 +0,0 @@ -use std::env; - -fn main() { - if let Ok(dir) = env::var("DEP_LLAMA_BACKENDS_DIR") { - println!("cargo:rustc-env=GGML_BACKENDS_DIR={dir}"); - } -} diff --git a/llama-cpp-bindings/src/lib.rs b/llama-cpp-bindings/src/lib.rs index 1f92378e5..17f1f9578 100644 --- a/llama-cpp-bindings/src/lib.rs +++ b/llama-cpp-bindings/src/lib.rs @@ -50,11 +50,7 @@ pub mod llama_token_attr; pub mod llama_token_attrs; pub mod llama_token_attrs_from_int_error; pub mod llguidance_sampler; -#[cfg(feature = "dynamic-backends")] -pub mod load_backends; -#[cfg(feature = "dynamic-backends")] pub mod load_backends_error; -#[cfg(feature = "dynamic-backends")] pub mod load_backends_from_path; pub mod log_options; pub mod log_record; diff --git a/llama-cpp-bindings/src/load_backends.rs b/llama-cpp-bindings/src/load_backends.rs deleted file mode 100644 index 0398c287e..000000000 --- a/llama-cpp-bindings/src/load_backends.rs +++ /dev/null @@ -1,28 +0,0 @@ -use std::path::Path; - -use crate::load_backends_error::LoadBackendsError; -use crate::load_backends_from_path::load_backends_from_path; - -pub const BACKENDS_DIR: Option<&str> = option_env!("GGML_BACKENDS_DIR"); - -/// # Errors -/// -/// Returns [`LoadBackendsError::PathNotUtf8`] when `BACKENDS_DIR` cannot be converted to UTF-8 -/// and [`LoadBackendsError::PathNullByte`] when it contains an interior null byte. -pub fn load_backends() -> Result<(), LoadBackendsError> { - let Some(dir) = BACKENDS_DIR else { - return Ok(()); - }; - - load_backends_from_path(Path::new(dir)) -} - -#[cfg(test)] -mod tests { - use super::load_backends; - - #[test] - fn load_backends_does_not_error_with_default_backends_dir() { - assert_eq!(load_backends(), Ok(())); - } -} diff --git a/llama-cpp-bindings/src/model/params.rs b/llama-cpp-bindings/src/model/params.rs index 79f2805ff..6f2de78f8 100644 --- a/llama-cpp-bindings/src/model/params.rs +++ b/llama-cpp-bindings/src/model/params.rs @@ -499,8 +499,6 @@ mod tests { use crate::error::llama_cpp_error::LlamaCppError; use crate::error::model_params_error::ModelParamsError; use crate::llama_backend::LlamaBackend; - #[cfg(feature = "dynamic-backends")] - use crate::load_backends::load_backends; use crate::max_devices::max_devices; use crate::model::llama_lazy_mode::LlamaLazyMode; use crate::model::llama_lazy_mode_parse_error::LlamaLazyModeParseError; @@ -787,7 +785,9 @@ mod tests { #[test] fn with_devices_sets_devices_when_available() { #[cfg(feature = "dynamic-backends")] - load_backends().unwrap(); + unsafe { + llama_cpp_bindings_sys::ggml_backend_load_all(); + } let dev_count = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_count() }; assert!(dev_count > 0, "Test requires at least one backend device"); From 9e7206d222b529ad563b61b38065ce5ceaf657b3 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 08:44:27 +0100 Subject: [PATCH 66/82] Build llama.cpp with one Release profile constant instead of mapping cargo profiles --- llama-cpp-bindings-build/src/build_context.rs | 25 +------------------ llama-cpp-bindings-build/src/cmake_build.rs | 4 +-- llama-cpp-bindings-build/src/lib.rs | 1 + .../src/link_directives.rs | 3 ++- llama-cpp-bindings-build/src/native_build.rs | 2 -- 5 files changed, 6 insertions(+), 29 deletions(-) diff --git a/llama-cpp-bindings-build/src/build_context.rs b/llama-cpp-bindings-build/src/build_context.rs index cd272fc01..496ea90b3 100644 --- a/llama-cpp-bindings-build/src/build_context.rs +++ b/llama-cpp-bindings-build/src/build_context.rs @@ -7,13 +7,6 @@ use crate::cargo_directive::CargoDirective; use crate::host_os::HostOs; use crate::target_os::TargetOs; -fn native_profile(cargo_profile: &str) -> String { - match cargo_profile { - "debug" | "release" => "Release".to_owned(), - other => other.to_owned(), - } -} - /// Where and for what the native libraries are built, as cargo describes the build script run. #[derive(Clone, Debug, Eq, PartialEq)] pub struct BuildContext { @@ -25,7 +18,6 @@ pub struct BuildContext { pub cargo_cfg_target_arch: String, pub cargo_cfg_target_env: String, pub build_shared_libs: bool, - pub profile: String, pub opt_level: String, pub debug_info: String, pub include_debug_messages: bool, @@ -58,7 +50,6 @@ impl BuildContext { .optional("CARGO_CFG_TARGET_ENV")? .unwrap_or_default(), build_shared_libs: environment.has(BuildFeature::DynamicLink), - profile: native_profile(&environment.required("PROFILE")?), opt_level: environment.required("OPT_LEVEL")?, debug_info: environment.required("DEBUG")?, include_debug_messages: environment.optional("BUILD_DEBUG")?.is_some(), @@ -100,7 +91,6 @@ pub mod tests { use enumflags2::BitFlags; use super::BuildContext; - use super::native_profile; use crate::android_ndk::tests::FIXTURE_ANDROID_NDK; use crate::build_environment::BuildEnvironment; use crate::build_environment::tests::environment_of; @@ -110,7 +100,7 @@ pub mod tests { use crate::host_os::HostOs; use crate::target_os::TargetOs; - const CARGO_VARIABLES: [(&str, &str); 10] = [ + const CARGO_VARIABLES: [(&str, &str); 9] = [ ("CARGO_CFG_TARGET_ARCH", "x86_64"), ("CARGO_CFG_TARGET_ENV", "gnu"), ("CARGO_CFG_TARGET_OS", "linux"), @@ -119,7 +109,6 @@ pub mod tests { ("HOST", "x86_64-unknown-linux-gnu"), ("OPT_LEVEL", "3"), ("OUT_DIR", "/out"), - ("PROFILE", "release"), ("TARGET", "x86_64-unknown-linux-gnu"), ]; @@ -153,17 +142,6 @@ pub mod tests { BuildContext::from_environment(environment, HostOs::Linux).unwrap() } - #[test] - fn cargo_debug_and_release_profiles_both_use_optimized_native_code() { - assert_eq!(native_profile("debug"), "Release"); - assert_eq!(native_profile("release"), "Release"); - } - - #[test] - fn custom_cargo_profile_name_is_preserved_for_cmake() { - assert_eq!(native_profile("RelWithDebInfo"), "RelWithDebInfo"); - } - #[test] fn the_context_is_read_from_the_build_variables_and_the_dynamic_link_feature() { assert_eq!( @@ -180,7 +158,6 @@ pub mod tests { cargo_cfg_target_arch: "x86_64".to_owned(), cargo_cfg_target_env: "gnu".to_owned(), build_shared_libs: true, - profile: "Release".to_owned(), opt_level: "3".to_owned(), debug_info: "false".to_owned(), include_debug_messages: true, diff --git a/llama-cpp-bindings-build/src/cmake_build.rs b/llama-cpp-bindings-build/src/cmake_build.rs index 805413013..e81efed61 100644 --- a/llama-cpp-bindings-build/src/cmake_build.rs +++ b/llama-cpp-bindings-build/src/cmake_build.rs @@ -5,13 +5,13 @@ use cmake::Config; use crate::build_error::BuildError; use crate::cmake_define::CmakeDefine; +use crate::native_profile::NATIVE_PROFILE; /// Configures the `CMake` project in `source_dir` and builds and installs it into `out_dir`. #[derive(Clone, Debug, Eq, PartialEq)] pub struct CmakeBuild { pub source_dir: PathBuf, pub out_dir: PathBuf, - pub profile: String, pub target_triple: String, pub host_triple: String, pub defines: Vec, @@ -45,7 +45,7 @@ impl CmakeBuild { .target(&self.target_triple) .host(&self.host_triple) .out_dir(&self.out_dir) - .profile(&self.profile) + .profile(NATIVE_PROFILE) .build(); Ok(()) diff --git a/llama-cpp-bindings-build/src/lib.rs b/llama-cpp-bindings-build/src/lib.rs index 6e72f0022..6ba978fd6 100644 --- a/llama-cpp-bindings-build/src/lib.rs +++ b/llama-cpp-bindings-build/src/lib.rs @@ -23,6 +23,7 @@ mod link_directives; mod link_kind; mod native_build; mod native_library; +mod native_profile; mod privatized_field; mod rebuild_triggers; mod target_os; diff --git a/llama-cpp-bindings-build/src/link_directives.rs b/llama-cpp-bindings-build/src/link_directives.rs index c80b79ccd..c0b114114 100644 --- a/llama-cpp-bindings-build/src/link_directives.rs +++ b/llama-cpp-bindings-build/src/link_directives.rs @@ -8,6 +8,7 @@ use crate::build_feature::BuildFeature; use crate::cargo_directive::CargoDirective; use crate::link_kind::LinkKind; use crate::native_library::NativeLibrary; +use crate::native_profile::NATIVE_PROFILE; use crate::target_os::TargetOs; const fn link(link_kind: LinkKind, name: &'static str) -> CargoDirective { @@ -62,7 +63,7 @@ fn private_dependency_search_paths(context: &BuildContext) -> Vec Date: Mon, 2 Nov 2026 08:45:25 +0100 Subject: [PATCH 67/82] Compile the wrappers with the C++ standard required rather than only when supported --- llama-cpp-bindings-build/src/wrapper_compilation.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llama-cpp-bindings-build/src/wrapper_compilation.rs b/llama-cpp-bindings-build/src/wrapper_compilation.rs index c244f3f75..948388908 100644 --- a/llama-cpp-bindings-build/src/wrapper_compilation.rs +++ b/llama-cpp-bindings-build/src/wrapper_compilation.rs @@ -20,7 +20,7 @@ pub struct WrapperCompilation { pub sources: Vec, pub include_dirs: Vec, pub flags: Vec, - pub cpp_standard_flag: String, + pub cpp_standard: &'static str, pub defines: Vec<&'static str>, pub cpp_stdlib_linked_separately: bool, pub target_triple: String, @@ -43,7 +43,7 @@ impl WrapperCompilation { .map(|include_dir| context.sys_dir.join(include_dir)) .collect(), flags: system_include_flags(&context.sys_dir), - cpp_standard_flag: format!("-std={CPP_STANDARD}"), + cpp_standard: CPP_STANDARD, defines: if environment.has(BuildFeature::FaultInjection) { vec![FAULT_INJECTION_DEFINE] } else { @@ -71,7 +71,7 @@ impl WrapperCompilation { build.flag(flag); } - build.flag_if_supported(&self.cpp_standard_flag); + build.std(self.cpp_standard); for define in &self.defines { build.define(define, None); @@ -125,7 +125,7 @@ mod tests { .collect(), include_dirs: vec![PathBuf::from("/sys/.")], flags: system_include_flags(Path::new("/sys")), - cpp_standard_flag: "-std=c++17".to_owned(), + cpp_standard: "c++17", defines: Vec::new(), cpp_stdlib_linked_separately: false, target_triple: "x86_64-unknown-linux-gnu".to_owned(), From 8278eadc13854005347669ddddd63e5974f59698 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 08:45:34 +0100 Subject: [PATCH 68/82] Use the C++ standard in the native build fixture --- llama-cpp-bindings-build/src/native_build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama-cpp-bindings-build/src/native_build.rs b/llama-cpp-bindings-build/src/native_build.rs index da28b8b10..b342b68f7 100644 --- a/llama-cpp-bindings-build/src/native_build.rs +++ b/llama-cpp-bindings-build/src/native_build.rs @@ -116,7 +116,7 @@ mod tests { sources: vec![PathBuf::from(FIXTURES).join("wrapper/wrapper_fixture.cpp")], include_dirs: vec![PathBuf::from(FIXTURES).join("wrapper")], flags: vec![format!("-isystem{FIXTURES}/wrapper/vendor")], - cpp_standard_flag: "-std=c++17".to_owned(), + cpp_standard: "c++17", defines: vec!["WRAPPER_FIXTURE_DEFINE"], cpp_stdlib_linked_separately: false, target_triple: HOST_TRIPLE.to_owned(), From a38ea1cfd2a67e03c2fb9f9edb2f6f68e705c3ed Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 08:45:40 +0100 Subject: [PATCH 69/82] Match Apple Silicon by the aarch64 target arch Rust actually reports --- llama-cpp-bindings/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama-cpp-bindings/Cargo.toml b/llama-cpp-bindings/Cargo.toml index bb140b710..36d42e8eb 100644 --- a/llama-cpp-bindings/Cargo.toml +++ b/llama-cpp-bindings/Cargo.toml @@ -43,7 +43,7 @@ android-static-stdcxx = ["llama-cpp-bindings-sys/static-stdcxx"] system-ggml = ["llama-cpp-bindings-sys/system-ggml"] system-ggml-static = ["system-ggml", "llama-cpp-bindings-sys/system-ggml-static"] -[target.'cfg(all(target_os = "macos", any(target_arch = "aarch64", target_arch = "arm64")))'.dependencies] +[target.'cfg(all(target_os = "macos", target_arch = "aarch64"))'.dependencies] llama-cpp-bindings-sys = { workspace = true, features = ["metal"] } [lints] From b1d2092e90e0bcb5908891f5abbded0748795ae4 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 08:47:36 +0100 Subject: [PATCH 70/82] Keep the device list NULL-terminated, store KV override keys byte for byte, and refuse keys llama.cpp cannot hold --- .../src/native_profile.rs | 1 + .../src/error/model_params_error.rs | 7 +- llama-cpp-bindings/src/model/params.rs | 164 ++++++++---------- .../src/model/params/empty_kv_override.rs | 9 + .../src/model/params/param_override_value.rs | 21 ++- 5 files changed, 95 insertions(+), 107 deletions(-) create mode 100644 llama-cpp-bindings-build/src/native_profile.rs create mode 100644 llama-cpp-bindings/src/model/params/empty_kv_override.rs diff --git a/llama-cpp-bindings-build/src/native_profile.rs b/llama-cpp-bindings-build/src/native_profile.rs new file mode 100644 index 000000000..2186083d7 --- /dev/null +++ b/llama-cpp-bindings-build/src/native_profile.rs @@ -0,0 +1 @@ +pub const NATIVE_PROFILE: &str = "Release"; diff --git a/llama-cpp-bindings/src/error/model_params_error.rs b/llama-cpp-bindings/src/error/model_params_error.rs index 8e70ebb4f..a8ad8ee1b 100644 --- a/llama-cpp-bindings/src/error/model_params_error.rs +++ b/llama-cpp-bindings/src/error/model_params_error.rs @@ -4,6 +4,9 @@ pub enum ModelParamsError { NoAvailableSlot, #[error("Override slot is not empty")] SlotNotEmpty, - #[error("Invalid character in key: byte {byte}, {reason}")] - InvalidCharacterInKey { byte: u8, reason: String }, + #[error("the key needs {length_with_nul} bytes with its NUL, but llama.cpp holds {capacity}")] + KeyTooLong { + length_with_nul: usize, + capacity: usize, + }, } diff --git a/llama-cpp-bindings/src/model/params.rs b/llama-cpp-bindings/src/model/params.rs index 6f2de78f8..78c2ca188 100644 --- a/llama-cpp-bindings/src/model/params.rs +++ b/llama-cpp-bindings/src/model/params.rs @@ -20,11 +20,13 @@ use crate::model::llama_lazy_mode_parse_error::LlamaLazyModeParseError; use crate::model::llama_load_mode::LlamaLoadMode; use crate::model::llama_load_mode_parse_error::LlamaLoadModeParseError; use crate::model::llama_split_mode_parse_error::LlamaSplitModeParseError; +use crate::model::params::empty_kv_override::EMPTY_KV_OVERRIDE; use crate::model::params::fit_extra_model::FitExtraModel; use crate::model::params::fit_result::FitResult; use crate::model::params::kv_overrides::KvOverrides; use crate::model::split_mode::LlamaSplitMode; +pub mod empty_kv_override; pub mod fit_extra_model; pub mod fit_result; pub mod kv_override_entry; @@ -33,8 +35,6 @@ pub mod kv_overrides; pub mod param_override_value; pub mod unknown_kv_override_tag; -pub const LLAMA_CPP_MAX_DEVICES: usize = 16; - fn fit_params_status_to_result( status: llama_cpp_bindings_sys::llama_rs_fit_params_status, out_unrecognized_status_code: i32, @@ -130,7 +130,7 @@ pub struct LlamaModelParams { kv_overrides: Vec, buft_overrides: Vec, buft_override_patterns: Vec, - devices: Pin>, + devices: Pin>, tensor_split: Vec, } @@ -144,7 +144,7 @@ impl Debug for LlamaModelParams { .field("load_mode", &self.load_mode()) .field("load_mtp", &self.params.load_mtp) .field("split_mode", &self.split_mode()) - .field("devices", &self.devices) + .field("devices", &self.devices()) .field("kv_overrides", &"vec of kv_overrides") .finish_non_exhaustive() } @@ -174,13 +174,17 @@ impl LlamaModelParams { return Err(ModelParamsError::SlotNotEmpty); } - for (i, &byte) in key.to_bytes_with_nul().iter().enumerate() { - kv_override.key[i] = c_char::try_from(byte).map_err(|convert_error| { - ModelParamsError::InvalidCharacterInKey { - byte, - reason: convert_error.to_string(), - } - })?; + let key_with_nul = key.to_bytes_with_nul(); + + if key_with_nul.len() > kv_override.key.len() { + return Err(ModelParamsError::KeyTooLong { + length_with_nul: key_with_nul.len(), + capacity: kv_override.key.len(), + }); + } + + for (slot, &byte) in kv_override.key.iter_mut().zip(key_with_nul) { + *slot = c_char::from_ne_bytes([byte]); } kv_override.tag = value.tag(); @@ -194,14 +198,7 @@ impl LlamaModelParams { fn push_kv_override_terminator(mut self: Pin<&mut Self>) { self.params.kv_overrides = null(); - self.kv_overrides - .push(llama_cpp_bindings_sys::llama_model_kv_override { - key: [0; 128], - tag: 0, - __bindgen_anon_1: llama_cpp_bindings_sys::llama_model_kv_override__bindgen_ty_1 { - val_i64: 0, - }, - }); + self.kv_overrides.push(EMPTY_KV_OVERRIDE); self.params.kv_overrides = self.kv_overrides.as_ptr(); } @@ -231,15 +228,6 @@ impl LlamaModelParams { return Err(ModelParamsError::SlotNotEmpty); } - for &byte in key.to_bytes_with_nul() { - c_char::try_from(byte).map_err(|convert_error| { - ModelParamsError::InvalidCharacterInKey { - byte, - reason: convert_error.to_string(), - } - })?; - } - let pattern = key.to_owned(); buft_override.pattern = pattern.as_ptr(); @@ -371,7 +359,7 @@ impl LlamaModelParams { for dev in self.devices.iter_mut() { *dev = ptr::null_mut(); } - let max_devices = max_devices().min(LLAMA_CPP_MAX_DEVICES); + let max_devices = max_devices(); if devices.len() > max_devices { return Err(LlamaCppError::MaxDevicesExceeded(max_devices)); } @@ -466,19 +454,13 @@ impl Default for LlamaModelParams { let default_params = unsafe { llama_cpp_bindings_sys::llama_model_default_params() }; Self { params: default_params, - kv_overrides: vec![llama_cpp_bindings_sys::llama_model_kv_override { - key: [0; 128], - tag: 0, - __bindgen_anon_1: llama_cpp_bindings_sys::llama_model_kv_override__bindgen_ty_1 { - val_i64: 0, - }, - }], + kv_overrides: vec![EMPTY_KV_OVERRIDE], buft_overrides: vec![llama_cpp_bindings_sys::llama_model_tensor_buft_override { pattern: null(), buft: ptr::null_mut(), }], buft_override_patterns: Vec::new(), - devices: Box::pin([ptr::null_mut(); 16]), + devices: Pin::new(vec![ptr::null_mut(); max_devices() + 1].into_boxed_slice()), tensor_split: Vec::new(), } } @@ -486,13 +468,12 @@ impl Default for LlamaModelParams { #[cfg(test)] mod tests { + use crate::model::params::empty_kv_override::KV_OVERRIDE_TEXT_CAPACITY; use std::ffi::CStr; use std::ffi::CString; - use std::ffi::c_char; use std::pin::pin; use std::ptr; - use super::LLAMA_CPP_MAX_DEVICES; use super::LlamaModelParams; use crate::context::params::LlamaContextParams; use crate::error::fit_error::FitError; @@ -506,17 +487,6 @@ mod tests { use crate::model::params::param_override_value::ParamOverrideValue; use crate::model::split_mode::LlamaSplitMode; - #[test] - fn device_capacity_is_not_smaller_than_the_upstream_limit() { - let upstream_limit = max_devices(); - assert!( - LLAMA_CPP_MAX_DEVICES >= upstream_limit, - "LLAMA_CPP_MAX_DEVICES ({LLAMA_CPP_MAX_DEVICES}) backs a fixed-size device array and is \ - clamped against llama_max_devices() ({upstream_limit}); once upstream reports more \ - devices than the array can hold, the extra devices are dropped without any error" - ); - } - #[test] fn default_params_match_the_llama_cpp_model_defaults() { let params = LlamaModelParams::default(); @@ -653,8 +623,7 @@ mod tests { format!("{params:?}"), "LlamaModelParams { n_gpu_layers: -1, main_gpu: 0, vocab_only: false, \ lazy_mode: Ok(On), load_mode: Ok(Auto), load_mtp: false, \ - split_mode: Ok(Layer), devices: [0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, \ - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0], \ + split_mode: Ok(Layer), devices: [], \ kv_overrides: \"vec of kv_overrides\", .. }" ); } @@ -768,17 +737,54 @@ mod tests { } #[test] - fn with_devices_too_many_returns_max_exceeded() { - let too_many: Vec = (0..17).collect(); - let result = LlamaModelParams::default().with_devices(&too_many); - let device_limit = max_devices().min(LLAMA_CPP_MAX_DEVICES); + fn more_devices_than_llama_cpp_supports_are_refused() { + let too_many = vec![0; max_devices() + 1]; - assert!( - matches!( - result, - Err(LlamaCppError::MaxDevicesExceeded(max_devices)) if max_devices == device_limit - ), - "17 devices must exceed the device limit of {device_limit}, got {result:?}" + assert_eq!( + LlamaModelParams::default().with_devices(&too_many).err(), + Some(LlamaCppError::MaxDevicesExceeded(max_devices())) + ); + } + + #[test] + fn a_full_device_list_stays_null_terminated() { + let params = LlamaModelParams::default() + .with_devices(&vec![0; max_devices()]) + .unwrap(); + + assert_eq!(params.devices.len(), max_devices() + 1); + assert_eq!(params.devices.last(), Some(&ptr::null_mut())); + } + + #[test] + fn a_kv_override_key_of_any_bytes_is_stored_as_given() { + let key = c"cl\xc3\xa9"; + let mut params = pin!(LlamaModelParams::default()); + + params + .as_mut() + .append_kv_override(key, ParamOverrideValue::Int(1)) + .unwrap(); + + assert_eq!( + unsafe { CStr::from_ptr(params.kv_overrides[0].key.as_ptr()) }, + key + ); + } + + #[test] + fn a_kv_override_key_longer_than_llama_cpp_holds_is_refused() { + let key = CString::new("k".repeat(KV_OVERRIDE_TEXT_CAPACITY)).unwrap(); + let mut params = pin!(LlamaModelParams::default()); + + assert_eq!( + params + .as_mut() + .append_kv_override(&key, ParamOverrideValue::Int(1)), + Err(ModelParamsError::KeyTooLong { + length_with_nul: KV_OVERRIDE_TEXT_CAPACITY + 1, + capacity: KV_OVERRIDE_TEXT_CAPACITY, + }) ); } @@ -811,40 +817,6 @@ mod tests { ); } - #[test] - fn append_kv_override_with_high_byte_returns_invalid_character_error() { - let key_bytes: &[u8] = b"\xff\0"; - let key = CStr::from_bytes_with_nul(key_bytes).unwrap(); - let mut params = pin!(LlamaModelParams::default()); - let result = params - .as_mut() - .append_kv_override(key, ParamOverrideValue::Int(1)); - - assert_eq!( - result, - Err(ModelParamsError::InvalidCharacterInKey { - byte: 0xff, - reason: c_char::try_from(0xff_u8).unwrap_err().to_string(), - }) - ); - } - - #[test] - fn add_cpu_buft_override_with_high_byte_returns_invalid_character_error() { - let key_bytes: &[u8] = b"\xff\0"; - let key = CStr::from_bytes_with_nul(key_bytes).unwrap(); - let mut params = pin!(LlamaModelParams::default()); - let result = params.as_mut().add_cpu_buft_override(key); - - assert_eq!( - result, - Err(ModelParamsError::InvalidCharacterInKey { - byte: 0xff, - reason: c_char::try_from(0xff_u8).unwrap_err().to_string(), - }) - ); - } - #[test] fn append_kv_override_with_empty_slot_vector_returns_no_available_slot() { let mut params = LlamaModelParams::default(); diff --git a/llama-cpp-bindings/src/model/params/empty_kv_override.rs b/llama-cpp-bindings/src/model/params/empty_kv_override.rs new file mode 100644 index 000000000..92f68fefe --- /dev/null +++ b/llama-cpp-bindings/src/model/params/empty_kv_override.rs @@ -0,0 +1,9 @@ +use std::mem; + +/// A KV override with every byte zero, which llama.cpp reads as the end of an override list. +pub const EMPTY_KV_OVERRIDE: llama_cpp_bindings_sys::llama_model_kv_override = + unsafe { mem::zeroed() }; + +/// How many bytes, its NUL included, a string KV override value holds in llama.cpp. +pub const KV_OVERRIDE_TEXT_CAPACITY: usize = + unsafe { EMPTY_KV_OVERRIDE.__bindgen_anon_1.val_str }.len(); diff --git a/llama-cpp-bindings/src/model/params/param_override_value.rs b/llama-cpp-bindings/src/model/params/param_override_value.rs index a6e13b55b..c7a47f627 100644 --- a/llama-cpp-bindings/src/model/params/param_override_value.rs +++ b/llama-cpp-bindings/src/model/params/param_override_value.rs @@ -1,5 +1,6 @@ use std::ffi::c_char; +use crate::model::params::empty_kv_override::KV_OVERRIDE_TEXT_CAPACITY; use crate::model::params::unknown_kv_override_tag::UnknownKvOverrideTag; #[derive(Debug, Clone, Copy, PartialEq)] @@ -7,7 +8,7 @@ pub enum ParamOverrideValue { Bool(bool), Float(f64), Int(i64), - Str([c_char; 128]), + Str([c_char; KV_OVERRIDE_TEXT_CAPACITY]), } impl ParamOverrideValue { @@ -73,6 +74,8 @@ mod tests { use std::mem; use super::ParamOverrideValue; + use crate::model::params::empty_kv_override::EMPTY_KV_OVERRIDE; + use crate::model::params::empty_kv_override::KV_OVERRIDE_TEXT_CAPACITY; use crate::model::params::unknown_kv_override_tag::UnknownKvOverrideTag; #[test] @@ -82,7 +85,7 @@ mod tests { ParamOverrideValue::Bool(true), ParamOverrideValue::Float(1.23), ParamOverrideValue::Int(42), - ParamOverrideValue::Str([0; 128]), + ParamOverrideValue::Str([0; KV_OVERRIDE_TEXT_CAPACITY]), ] .map(|value| value.tag()), [ @@ -124,11 +127,11 @@ mod tests { #[test] fn an_int_kv_override_reads_as_an_int_value() { let ffi_override = llama_cpp_bindings_sys::llama_model_kv_override { - key: [0; 128], tag: llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_INT, __bindgen_anon_1: llama_cpp_bindings_sys::llama_model_kv_override__bindgen_ty_1 { val_i64: 123, }, + ..EMPTY_KV_OVERRIDE }; let value = ParamOverrideValue::try_from(&ffi_override).unwrap(); @@ -139,11 +142,11 @@ mod tests { #[test] fn a_float_kv_override_reads_as_a_float_value() { let ffi_override = llama_cpp_bindings_sys::llama_model_kv_override { - key: [0; 128], tag: llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_FLOAT, __bindgen_anon_1: llama_cpp_bindings_sys::llama_model_kv_override__bindgen_ty_1 { val_f64: 1.5, }, + ..EMPTY_KV_OVERRIDE }; let value = ParamOverrideValue::try_from(&ffi_override).unwrap(); @@ -154,11 +157,11 @@ mod tests { #[test] fn a_bool_kv_override_reads_as_a_bool_value() { let ffi_override = llama_cpp_bindings_sys::llama_model_kv_override { - key: [0; 128], tag: llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_BOOL, __bindgen_anon_1: llama_cpp_bindings_sys::llama_model_kv_override__bindgen_ty_1 { val_bool: false, }, + ..EMPTY_KV_OVERRIDE }; let value = ParamOverrideValue::try_from(&ffi_override).unwrap(); @@ -168,7 +171,7 @@ mod tests { #[test] fn a_str_override_stores_its_bytes_in_the_str_field() { - let mut str_data = [0i8; 128]; + let mut str_data = [0; KV_OVERRIDE_TEXT_CAPACITY]; str_data[0] = b'h'.cast_signed(); str_data[1] = b'i'.cast_signed(); @@ -182,16 +185,16 @@ mod tests { #[test] fn a_str_kv_override_reads_as_a_str_value() { - let mut str_data = [0i8; 128]; + let mut str_data = [0; KV_OVERRIDE_TEXT_CAPACITY]; str_data[0] = b'a'.cast_signed(); str_data[1] = b'b'.cast_signed(); let ffi_override = llama_cpp_bindings_sys::llama_model_kv_override { - key: [0; 128], tag: llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_STR, __bindgen_anon_1: llama_cpp_bindings_sys::llama_model_kv_override__bindgen_ty_1 { val_str: str_data, }, + ..EMPTY_KV_OVERRIDE }; let value = ParamOverrideValue::try_from(&ffi_override).unwrap(); @@ -202,9 +205,9 @@ mod tests { #[test] fn an_unknown_kv_override_tag_is_reported_with_its_value() { let ffi_override = llama_cpp_bindings_sys::llama_model_kv_override { - key: [0; 128], tag: 9999, __bindgen_anon_1: unsafe { mem::zeroed() }, + ..EMPTY_KV_OVERRIDE }; assert_eq!( From 6ba6ab55fdf61203e07cf696d08e5f76152158a8 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 08:47:52 +0100 Subject: [PATCH 71/82] Run every non-model crate's tests in test.unit and rebuild the compile database when any wrapper-sources file changes --- Makefile | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 7d65ee485..e8b9dc16e 100644 --- a/Makefile +++ b/Makefile @@ -19,12 +19,8 @@ WRAPPER_SOURCES_RESPONSE_FILE = target/wrapper_sources.rsp WRAPPER_SOURCES_CRATE = llama-cpp-wrapper-sources WRAPPER_SOURCES_CRATE_FILES = \ - $(WRAPPER_SOURCES_CRATE)/src/compile_commands_file.rs \ - $(WRAPPER_SOURCES_CRATE)/src/cpp_standard.rs \ - $(WRAPPER_SOURCES_CRATE)/src/wrapper_include_dirs.rs \ - $(WRAPPER_SOURCES_CRATE)/src/wrapper_source_paths.rs \ - $(WRAPPER_SOURCES_CRATE)/src/wrapper_sources.rs \ - $(WRAPPER_SOURCES_CRATE)/src/wrapper_sources_response_file.rs + $(WRAPPER_SOURCES_CRATE)/Cargo.toml \ + $(wildcard $(WRAPPER_SOURCES_CRATE)/src/*.rs) EMIT_WRAPPER_BUILD_INPUTS = cargo run --quiet --package $(WRAPPER_SOURCES_CRATE) -- \ $(CURDIR)/llama-cpp-bindings-sys $(COMPILE_COMMANDS) $(WRAPPER_SOURCES_RESPONSE_FILE) @@ -134,5 +130,6 @@ test.llms: clippy test.harness test.unit .PHONY: test.unit test.unit: clippy - cargo test -p llama-cpp-bindings -p llama-cpp-bindings-build -p llama-cpp-gbnf \ + cargo test -p llama-cpp-bindings -p llama-cpp-bindings-build -p llama-cpp-bindings-types \ + -p llama-cpp-error-recorder -p llama-cpp-ffi-status -p llama-cpp-gbnf \ -p llama-cpp-log-decoder -p llama-cpp-wrapper-sources $(DEVICE_FEATURE) From f24c7f0528d5e69980fe0f6bc381b9f5896b2954 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 08:48:27 +0100 Subject: [PATCH 72/82] Request every GPU layer with -1 as llama.cpp defines and parse negative attribute literals as values --- .../approximate_tok_env_eos_selection.rs | 4 +- .../tests/backend_device_listing.rs | 4 +- .../tests/batch_context_limits.rs | 2 +- .../tests/chat_answer_parsing.rs | 30 +++--- .../tests/chat_prompt_rendering.rs | 24 ++--- .../tests/chat_session_input_validation.rs | 12 +-- .../tests/chat_session_llama_cpp_failures.rs | 20 ++-- .../tests/chat_template_lookup.rs | 18 ++-- .../tests/chat_template_override.rs | 6 +- .../tests/completion_usage_accounting.rs | 16 +-- .../tests/context_creation_limits.rs | 18 ++-- .../tests/context_dimensions.rs | 12 +-- .../tests/debug_formatting.rs | 4 +- .../tests/decode_abort_control.rs | 6 +- .../tests/decode_batch_rejection.rs | 8 +- .../tests/detokenization.rs | 46 ++++----- llama-cpp-bindings-tests/tests/dry_sampler.rs | 4 +- .../tests/embedding_availability.rs | 6 +- .../tests/embedding_generation.rs | 10 +- .../tests/embedding_retrieval_errors.rs | 8 +- .../tests/embedding_row_width.rs | 6 +- .../tests/embeddings_toggling.rs | 4 +- .../tests/encode_batch_rejection.rs | 4 +- .../tests/encoder_requirement.rs | 4 +- .../end_of_generation_token_detection.rs | 8 +- .../tests/generation_end_classification.rs | 6 +- .../tests/grammar_sampler.rs | 28 +++--- .../tests/image_question_answering.rs | 2 +- .../json_schema_constrained_generation.rs | 16 +-- .../tests/lazy_grammar_sampler.rs | 12 +-- .../tests/llguidance_sampler.rs | 98 +++++++++---------- .../tests/logit_bias_sampler.rs | 2 +- .../tests/logits_availability.rs | 8 +- .../tests/logits_retrieval.rs | 6 +- .../tests/lora_adapters.rs | 14 +-- .../tests/memory_availability.rs | 10 +- .../tests/memory_occupancy.rs | 24 ++--- .../tests/memory_position_bounds.rs | 64 ++++++------ ...emory_position_shift_rope_compatibility.rs | 16 +-- .../tests/memory_priming.rs | 6 +- .../tests/memory_sequence_edits.rs | 26 ++--- .../tests/memory_sequence_id_bounds.rs | 26 ++--- .../tests/model_architecture.rs | 24 ++--- .../tests/model_derived_data_caching.rs | 4 +- .../tests/model_hyperparameters.rs | 8 +- .../tests/model_loading_errors.rs | 8 +- .../tests/model_metadata.rs | 64 ++++++------ .../tests/multimodal_audio.rs | 10 +- .../tests/multimodal_bitmap_construction.rs | 8 +- .../tests/multimodal_chunk_copy.rs | 4 +- .../tests/multimodal_chunk_encoding.rs | 4 +- .../tests/multimodal_chunk_evaluation.rs | 58 +++++------ .../tests/multimodal_chunk_tokenization.rs | 42 ++++---- .../tests/multimodal_fusion.rs | 2 +- .../multimodal_projector_capabilities.rs | 22 ++--- .../tests/multimodal_projector_loading.rs | 6 +- .../multimodal_prompt_parts_tokenization.rs | 12 +-- .../multimodal_reasoning_classification.rs | 10 +- .../tests/multimodal_usage_accounting.rs | 12 +-- .../tests/parameter_fitting.rs | 12 +-- .../tests/prompt_token_accounting.rs | 38 +++---- .../tests/reasoning_classification.rs | 24 ++--- llama-cpp-bindings-tests/tests/reranking.rs | 2 +- .../tests/sampled_token_ingestion.rs | 34 +++---- .../tests/sampler_token_acceptance.rs | 4 +- .../tests/session_file_path_validation.rs | 64 ++++++------ .../tests/session_file_token_capacity.rs | 16 +-- .../tests/session_files.rs | 48 ++++----- .../tests/special_tokens.rs | 38 +++---- .../tests/state_snapshots.rs | 26 ++--- .../tests/token_attributes.rs | 8 +- .../tests/token_candidates.rs | 4 +- .../tests/tokenization.rs | 22 ++--- .../tests/tool_call_classification.rs | 2 +- .../tests/tool_call_grammar.rs | 8 +- .../tests/tool_call_parsing.rs | 20 ++-- .../tests/unconstrained_token_sampling.rs | 20 ++-- .../tests/vocabulary_bounds.rs | 4 +- .../tests/vocabulary_enumeration.rs | 2 +- .../tests/vocabulary_type.rs | 8 +- .../src/parsed_args.rs | 50 +++++++--- .../tests/phase_conclusions.rs | 18 ++-- 82 files changed, 720 insertions(+), 698 deletions(-) diff --git a/llama-cpp-bindings-tests/tests/approximate_tok_env_eos_selection.rs b/llama-cpp-bindings-tests/tests/approximate_tok_env_eos_selection.rs index 991afb27d..a52c15951 100644 --- a/llama-cpp-bindings-tests/tests/approximate_tok_env_eos_selection.rs +++ b/llama-cpp-bindings-tests/tests/approximate_tok_env_eos_selection.rs @@ -16,7 +16,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -41,7 +41,7 @@ fn approximate_tok_env_falls_back_to_eos_when_eot_unavailable( #[llama_test( model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/backend_device_listing.rs b/llama-cpp-bindings-tests/tests/backend_device_listing.rs index c59667d45..0e40ff916 100644 --- a/llama-cpp-bindings-tests/tests/backend_device_listing.rs +++ b/llama-cpp-bindings-tests/tests/backend_device_listing.rs @@ -17,7 +17,7 @@ fn listing_with_missing_text_after(passing_hits: usize) -> Option) -> Result

) -> Result

) -> Result) -> R #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -234,7 +234,7 @@ fn parses_an_answer_with_a_nul_byte_as_content(fixture: &LlamaFixture<'_>) -> Re #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -258,7 +258,7 @@ fn qwen35_chat_session_parses_every_answer_it_is_given(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs b/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs index 1b35b4256..c9b2f8e89 100644 --- a/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs +++ b/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs @@ -11,7 +11,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -19,7 +19,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -27,7 +27,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -35,7 +35,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -43,7 +43,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -54,7 +54,7 @@ use llama_cpp_test_harness_macros::llama_test; "unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf" ), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -73,7 +73,7 @@ fn chat_session_renders_the_user_message_into_the_prompt(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -81,7 +81,7 @@ fn chat_session_renders_the_user_message_into_the_prompt(fixture: &LlamaFixture< )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -89,7 +89,7 @@ fn chat_session_renders_the_user_message_into_the_prompt(fixture: &LlamaFixture< )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -97,7 +97,7 @@ fn chat_session_renders_the_user_message_into_the_prompt(fixture: &LlamaFixture< )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -116,7 +116,7 @@ fn chat_session_renders_long_messages_in_full(fixture: &LlamaFixture<'_>) -> Res #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -137,7 +137,7 @@ fn qwen35_generation_prompt_opens_the_assistant_turn_with_thinking( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs index 92e50aeb6..4098cad07 100644 --- a/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs +++ b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs @@ -18,7 +18,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 64, n_batch = 64, @@ -40,7 +40,7 @@ fn chat_message_reports_the_nul_byte_position_after_the_media_marker( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -68,7 +68,7 @@ fn chat_session_refuses_a_tool_missing_its_function_name(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -87,7 +87,7 @@ fn chat_session_refuses_tools_with_a_nul_byte(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -125,7 +125,7 @@ fn chat_session_refuses_a_message_that_is_not_utf8(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -169,7 +169,7 @@ fn tokenizing_a_prompt_whose_reasoning_prefill_holds_a_nul_character_is_refused( #[llama_test( model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs b/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs index 1fd2693eb..574501cbf 100644 --- a/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs +++ b/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs @@ -24,7 +24,7 @@ fn not_utf8() -> Utf8Error { #[llama_test( model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -43,7 +43,7 @@ fn a_chat_session_needs_a_model_with_a_chat_template(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, @@ -67,7 +67,7 @@ fn every_prompt_reader_refuses_prompt_text_that_is_not_utf8(fixture: &LlamaFixtu #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, @@ -90,7 +90,7 @@ fn grammar_triggers_refuse_a_trigger_value_that_is_not_utf8(fixture: &LlamaFixtu #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, @@ -117,7 +117,7 @@ fn grammar_triggers_refuse_a_trigger_type_the_bindings_do_not_recognize( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, @@ -136,7 +136,7 @@ fn the_grammar_sampler_refuses_a_grammar_that_is_not_utf8(fixture: &LlamaFixture #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, @@ -158,7 +158,7 @@ fn the_grammar_sampler_reports_a_grammar_llama_cpp_cannot_load(fixture: &LlamaFi #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, @@ -183,7 +183,7 @@ fn the_grammar_sampler_reports_a_generation_prompt_token_the_grammar_cannot_acce #[llama_test( model_source = HuggingFace("Qwen/Qwen2.5-7B-Instruct-GGUF", "qwen2.5-7b-instruct-q2_k.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -204,7 +204,7 @@ fn the_grammar_sampler_reports_a_trigger_word_llama_cpp_cannot_escape(fixture: & #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, @@ -226,7 +226,7 @@ fn the_grammar_sampler_reports_a_lazy_grammar_llama_cpp_cannot_load(fixture: &Ll #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/chat_template_lookup.rs b/llama-cpp-bindings-tests/tests/chat_template_lookup.rs index ae5d1d49f..100a2a265 100644 --- a/llama-cpp-bindings-tests/tests/chat_template_lookup.rs +++ b/llama-cpp-bindings-tests/tests/chat_template_lookup.rs @@ -8,7 +8,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -16,7 +16,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -24,7 +24,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -32,7 +32,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -46,7 +46,7 @@ fn default_chat_template_is_not_empty(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -54,7 +54,7 @@ fn default_chat_template_is_not_empty(fixture: &LlamaFixture<'_>) -> Result<()> )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -62,7 +62,7 @@ fn default_chat_template_is_not_empty(fixture: &LlamaFixture<'_>) -> Result<()> )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -70,7 +70,7 @@ fn default_chat_template_is_not_empty(fixture: &LlamaFixture<'_>) -> Result<()> )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -85,7 +85,7 @@ fn chat_template_with_nonexistent_name_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/chat_template_override.rs b/llama-cpp-bindings-tests/tests/chat_template_override.rs index 5b1b71235..9cb05a477 100644 --- a/llama-cpp-bindings-tests/tests/chat_template_override.rs +++ b/llama-cpp-bindings-tests/tests/chat_template_override.rs @@ -66,7 +66,7 @@ fn token_count(model: &LlamaModel, text: &str) -> Result { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 1024, n_batch = 128, @@ -97,7 +97,7 @@ fn deepseek_r1_served_with_the_llama_cpp_template_parses_its_tool_call( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 1024, n_batch = 128, @@ -135,7 +135,7 @@ fn deepseek_r1_served_with_the_llama_cpp_template_streams_its_tool_call_tokens_a #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 1024, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs index f104faaba..5e0688f88 100644 --- a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs +++ b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs @@ -19,7 +19,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -27,7 +27,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -35,7 +35,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -43,7 +43,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -138,7 +138,7 @@ fn raw_prompt_completion_usage_matches_the_classified_tokens( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, @@ -146,7 +146,7 @@ fn raw_prompt_completion_usage_matches_the_classified_tokens( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, @@ -154,7 +154,7 @@ fn raw_prompt_completion_usage_matches_the_classified_tokens( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, @@ -162,7 +162,7 @@ fn raw_prompt_completion_usage_matches_the_classified_tokens( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/context_creation_limits.rs b/llama-cpp-bindings-tests/tests/context_creation_limits.rs index 9e4f0508d..c4ad0c22a 100644 --- a/llama-cpp-bindings-tests/tests/context_creation_limits.rs +++ b/llama-cpp-bindings-tests/tests/context_creation_limits.rs @@ -11,7 +11,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4294967295, n_batch = 128, @@ -19,7 +19,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4294967295, n_batch = 128, @@ -27,7 +27,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4294967295, n_batch = 128, @@ -35,7 +35,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4294967295, n_batch = 128, @@ -50,7 +50,7 @@ fn a_context_size_overflowing_llama_cpp_padding_is_unconstructible(fixture: &Lla #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 1, @@ -76,7 +76,7 @@ fn new_context_rejects_more_sequences_than_one_batch_can_output(fixture: &LlamaF #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 2, @@ -103,7 +103,7 @@ fn new_context_rejects_more_sequences_than_a_causal_context_can_output(fixture: #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -127,7 +127,7 @@ fn creating_a_context_reports_a_batch_llama_cpp_cannot_allocate(fixture: &LlamaF #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -146,7 +146,7 @@ fn a_context_whose_pooling_type_llama_cpp_does_not_define_is_refused(fixture: &L #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/context_dimensions.rs b/llama-cpp-bindings-tests/tests/context_dimensions.rs index 0174e208b..0d84b760c 100644 --- a/llama-cpp-bindings-tests/tests/context_dimensions.rs +++ b/llama-cpp-bindings-tests/tests/context_dimensions.rs @@ -6,7 +6,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 128, @@ -14,7 +14,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 128, @@ -22,7 +22,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 128, @@ -30,7 +30,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 128, @@ -38,7 +38,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -60,7 +60,7 @@ fn context_reports_positive_context_and_batch_sizes(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/debug_formatting.rs b/llama-cpp-bindings-tests/tests/debug_formatting.rs index 810590689..fa7698519 100644 --- a/llama-cpp-bindings-tests/tests/debug_formatting.rs +++ b/llama-cpp-bindings-tests/tests/debug_formatting.rs @@ -6,7 +6,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -27,7 +27,7 @@ fn context_debug_format_contains_struct_name(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/decode_abort_control.rs b/llama-cpp-bindings-tests/tests/decode_abort_control.rs index 8a7da1161..3d8fb541c 100644 --- a/llama-cpp-bindings-tests/tests/decode_abort_control.rs +++ b/llama-cpp-bindings-tests/tests/decode_abort_control.rs @@ -14,7 +14,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -44,7 +44,7 @@ fn set_abort_flag_aborts_decode(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -72,7 +72,7 @@ fn set_abort_flag_false_allows_decode(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, diff --git a/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs b/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs index d0c579ed7..4e96409df 100644 --- a/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs +++ b/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs @@ -15,7 +15,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -36,7 +36,7 @@ fn decoding_an_empty_batch_is_refused_as_invalid(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -74,7 +74,7 @@ fn decoding_more_tokens_than_the_micro_batch_with_non_causal_attention_is_refuse #[llama_test( model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -107,7 +107,7 @@ fn decoding_more_tokens_than_the_micro_batch_on_an_encoder_is_refused( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, diff --git a/llama-cpp-bindings-tests/tests/detokenization.rs b/llama-cpp-bindings-tests/tests/detokenization.rs index 084e47cd2..69c0b306c 100644 --- a/llama-cpp-bindings-tests/tests/detokenization.rs +++ b/llama-cpp-bindings-tests/tests/detokenization.rs @@ -18,7 +18,7 @@ const PIECE_PROBE_BYTES: usize = 32; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -26,7 +26,7 @@ const PIECE_PROBE_BYTES: usize = 32; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -34,7 +34,7 @@ const PIECE_PROBE_BYTES: usize = 32; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -42,7 +42,7 @@ const PIECE_PROBE_BYTES: usize = 32; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -63,7 +63,7 @@ fn token_to_piece_bytes_returns_the_bytes_of_the_token_piece( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -71,7 +71,7 @@ fn token_to_piece_bytes_returns_the_bytes_of_the_token_piece( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -79,7 +79,7 @@ fn token_to_piece_bytes_returns_the_bytes_of_the_token_piece( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -87,7 +87,7 @@ fn token_to_piece_bytes_returns_the_bytes_of_the_token_piece( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -133,7 +133,7 @@ fn token_to_piece_returns_a_piece_longer_than_its_probe_buffer_whole( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -141,7 +141,7 @@ fn token_to_piece_returns_a_piece_longer_than_its_probe_buffer_whole( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -149,7 +149,7 @@ fn token_to_piece_returns_a_piece_longer_than_its_probe_buffer_whole( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -157,7 +157,7 @@ fn token_to_piece_returns_a_piece_longer_than_its_probe_buffer_whole( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -180,7 +180,7 @@ fn token_to_piece_bytes_with_a_too_small_buffer_reports_the_size_the_piece_needs #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -188,7 +188,7 @@ fn token_to_piece_bytes_with_a_too_small_buffer_reports_the_size_the_piece_needs )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -196,7 +196,7 @@ fn token_to_piece_bytes_with_a_too_small_buffer_reports_the_size_the_piece_needs )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -204,7 +204,7 @@ fn token_to_piece_bytes_with_a_too_small_buffer_reports_the_size_the_piece_needs )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -231,7 +231,7 @@ fn token_to_piece_strips_the_requested_count_of_leading_spaces( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -239,7 +239,7 @@ fn token_to_piece_strips_the_requested_count_of_leading_spaces( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -247,7 +247,7 @@ fn token_to_piece_strips_the_requested_count_of_leading_spaces( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -255,7 +255,7 @@ fn token_to_piece_strips_the_requested_count_of_leading_spaces( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -289,7 +289,7 @@ fn token_to_piece_decodes_a_token_alike_in_every_non_content_section(fixture: &L #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -311,7 +311,7 @@ fn token_to_piece_bytes_refuses_a_buffer_beyond_a_c_int_before_allocating( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -346,7 +346,7 @@ fn token_to_piece_replaces_a_lone_continuation_byte_with_the_replacement_charact #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/dry_sampler.rs b/llama-cpp-bindings-tests/tests/dry_sampler.rs index 01c1a430c..8477f9e2e 100644 --- a/llama-cpp-bindings-tests/tests/dry_sampler.rs +++ b/llama-cpp-bindings-tests/tests/dry_sampler.rs @@ -9,7 +9,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -27,7 +27,7 @@ fn building_a_dry_sampler_with_sequence_breakers_succeeds( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, diff --git a/llama-cpp-bindings-tests/tests/embedding_availability.rs b/llama-cpp-bindings-tests/tests/embedding_availability.rs index 9d78323fa..0a2bf16ee 100644 --- a/llama-cpp-bindings-tests/tests/embedding_availability.rs +++ b/llama-cpp-bindings-tests/tests/embedding_availability.rs @@ -6,7 +6,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -24,7 +24,7 @@ fn sequence_embeddings_need_a_pooling_context(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -44,7 +44,7 @@ fn token_embeddings_need_a_context_without_pooling(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/embedding_generation.rs b/llama-cpp-bindings-tests/tests/embedding_generation.rs index 47b67be6c..cbb1d0222 100644 --- a/llama-cpp-bindings-tests/tests/embedding_generation.rs +++ b/llama-cpp-bindings-tests/tests/embedding_generation.rs @@ -13,7 +13,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -56,7 +56,7 @@ fn an_embedding_pass_records_its_prompt_tokens_and_no_completion_tokens( #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -87,7 +87,7 @@ fn a_sequence_embedding_is_as_wide_as_the_model_embedding( #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -154,7 +154,7 @@ fn multi_sequence_embeddings_returns_one_embedding_per_sequence( #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -236,7 +236,7 @@ fn embeddings_returns_distinct_values_when_reused_batch_has_extra_capacity( #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, diff --git a/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs b/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs index 575a55998..e05d58c8e 100644 --- a/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs +++ b/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs @@ -12,7 +12,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -34,7 +34,7 @@ fn token_embedding_reports_embeddings_not_enabled_on_a_context_without_embedding #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -59,7 +59,7 @@ fn sequence_embedding_reports_embeddings_not_enabled_on_a_context_without_embedd #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -95,7 +95,7 @@ fn sequence_embedding_of_a_sequence_missing_from_the_decoded_batch_is_unavailabl #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, diff --git a/llama-cpp-bindings-tests/tests/embedding_row_width.rs b/llama-cpp-bindings-tests/tests/embedding_row_width.rs index 13ba21988..cc2650fc1 100644 --- a/llama-cpp-bindings-tests/tests/embedding_row_width.rs +++ b/llama-cpp-bindings-tests/tests/embedding_row_width.rs @@ -10,7 +10,7 @@ const QWEN3_RERANKER_CLASSIFIER_LABELS: usize = 2; #[llama_test( model_source = HuggingFace("LiquidAI/LFM2-ColBERT-350M-GGUF", "LFM2-ColBERT-350M-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -30,7 +30,7 @@ fn token_embeddings_are_as_wide_as_the_output_projection(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("LiquidAI/LFM2-ColBERT-350M-GGUF", "LFM2-ColBERT-350M-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -52,7 +52,7 @@ fn pooled_sequence_embeddings_are_as_wide_as_the_output_projection( #[llama_test( model_source = HuggingFace("ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF", "qwen3-reranker-0.6b-q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/embeddings_toggling.rs b/llama-cpp-bindings-tests/tests/embeddings_toggling.rs index 32807fa50..ab9d512b0 100644 --- a/llama-cpp-bindings-tests/tests/embeddings_toggling.rs +++ b/llama-cpp-bindings-tests/tests/embeddings_toggling.rs @@ -87,7 +87,7 @@ fn row_hidden_state_after_enabling_embeddings( #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -122,7 +122,7 @@ fn final_hidden_states_after_enabling_embeddings_for_a_row_equal_those_of_an_emb #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs b/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs index e263aef76..25ee00af5 100644 --- a/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs +++ b/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs @@ -13,7 +13,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -44,7 +44,7 @@ fn encoding_more_tokens_than_the_micro_batch_is_refused(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/encoder_requirement.rs b/llama-cpp-bindings-tests/tests/encoder_requirement.rs index a56834fa6..699fe54db 100644 --- a/llama-cpp-bindings-tests/tests/encoder_requirement.rs +++ b/llama-cpp-bindings-tests/tests/encoder_requirement.rs @@ -11,7 +11,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -40,7 +40,7 @@ fn encoding_with_a_decoder_only_model_reports_it_has_no_encoder( #[llama_test( model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, diff --git a/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs b/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs index ff2bdebd2..c621362f2 100644 --- a/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs +++ b/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs @@ -7,7 +7,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -15,7 +15,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -23,7 +23,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -31,7 +31,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/generation_end_classification.rs b/llama-cpp-bindings-tests/tests/generation_end_classification.rs index b32192d0d..e6db6af12 100644 --- a/llama-cpp-bindings-tests/tests/generation_end_classification.rs +++ b/llama-cpp-bindings-tests/tests/generation_end_classification.rs @@ -21,7 +21,7 @@ fn visible_text(outcomes: &[IngestOutcome]) -> String { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 128, @@ -57,7 +57,7 @@ fn visible_text_in(outcomes: &[IngestOutcome], section: SampledTokenSection) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 128, @@ -102,7 +102,7 @@ fn an_answer_stopped_inside_unclosed_reasoning_releases_its_unsettled_tail_as_co #[llama_test( model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/grammar_sampler.rs b/llama-cpp-bindings-tests/tests/grammar_sampler.rs index b2f238259..9dc7b7191 100644 --- a/llama-cpp-bindings-tests/tests/grammar_sampler.rs +++ b/llama-cpp-bindings-tests/tests/grammar_sampler.rs @@ -24,7 +24,7 @@ const YES_TOKEN_AND_END_OF_GENERATION: u64 = 2; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -32,7 +32,7 @@ const YES_TOKEN_AND_END_OF_GENERATION: u64 = 2; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -40,7 +40,7 @@ const YES_TOKEN_AND_END_OF_GENERATION: u64 = 2; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -48,7 +48,7 @@ const YES_TOKEN_AND_END_OF_GENERATION: u64 = 2; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -105,7 +105,7 @@ fn grammar_sampler_constrains_the_first_token_of_the_answer_to_yes( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -113,7 +113,7 @@ fn grammar_sampler_constrains_the_first_token_of_the_answer_to_yes( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -121,7 +121,7 @@ fn grammar_sampler_constrains_the_first_token_of_the_answer_to_yes( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -129,7 +129,7 @@ fn grammar_sampler_constrains_the_first_token_of_the_answer_to_yes( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -192,7 +192,7 @@ fn grammar_constrained_chat_answer_is_yes_content_that_ends_generation( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -217,7 +217,7 @@ fn a_grammar_sampler_masks_tokens_that_cannot_start_the_grammar( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 128, @@ -255,7 +255,7 @@ fn qwen35_grammar_with_a_token_reference_constrains_the_first_token( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 128, @@ -270,7 +270,7 @@ fn qwen35_grammar_without_its_root_reports_the_missing_root(fixture: &LlamaFixtu #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 128, @@ -287,7 +287,7 @@ fn qwen35_grammar_with_a_syntax_error_reports_it(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -304,7 +304,7 @@ fn a_grammar_with_a_nul_byte_is_refused(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/image_question_answering.rs b/llama-cpp-bindings-tests/tests/image_question_answering.rs index 0d28bc531..7cba5800d 100644 --- a/llama-cpp-bindings-tests/tests/image_question_answering.rs +++ b/llama-cpp-bindings-tests/tests/image_question_answering.rs @@ -19,7 +19,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4096, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs index ce6f06661..0895dceda 100644 --- a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs +++ b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs @@ -67,7 +67,7 @@ fn json_schema_constrained_answer(fixture: &LlamaFixture<'_>) -> Result { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -84,7 +84,7 @@ fn deepseek_r1_8b_answers_the_json_schema_with_four(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -101,7 +101,7 @@ fn glm47_flash_answers_the_json_schema_with_a_sentence(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -118,7 +118,7 @@ fn qwen35_answers_the_json_schema_with_four(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -135,7 +135,7 @@ fn qwen36_answers_the_json_schema_with_an_empty_string(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -143,7 +143,7 @@ fn qwen36_answers_the_json_schema_with_an_empty_string(fixture: &LlamaFixture<'_ )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -151,7 +151,7 @@ fn qwen36_answers_the_json_schema_with_an_empty_string(fixture: &LlamaFixture<'_ )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -159,7 +159,7 @@ fn qwen36_answers_the_json_schema_with_an_empty_string(fixture: &LlamaFixture<'_ )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs b/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs index 4e1b0c8e1..ee6b020cd 100644 --- a/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs +++ b/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs @@ -11,7 +11,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -37,7 +37,7 @@ fn a_lazy_grammar_awaiting_its_trigger_pattern_leaves_candidates_unconstrained( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -53,7 +53,7 @@ fn grammar_lazy_without_its_root_reports_the_missing_root(fixture: &LlamaFixture #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -74,7 +74,7 @@ fn grammar_lazy_refuses_a_trigger_pattern_with_a_nul_byte(fixture: &LlamaFixture #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -105,7 +105,7 @@ fn a_lazy_grammar_awaiting_its_trigger_token_leaves_candidates_unconstrained( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -125,7 +125,7 @@ fn grammar_lazy_refuses_a_grammar_with_a_nul_byte(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, diff --git a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs index 980ea6ec7..d97373ff9 100644 --- a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs +++ b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs @@ -49,7 +49,7 @@ fn yes_no_and_hello(model: &LlamaModel) -> [LlamaToken; 3] { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -57,7 +57,7 @@ fn yes_no_and_hello(model: &LlamaModel) -> [LlamaToken; 3] { )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -65,7 +65,7 @@ fn yes_no_and_hello(model: &LlamaModel) -> [LlamaToken; 3] { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -73,7 +73,7 @@ fn yes_no_and_hello(model: &LlamaModel) -> [LlamaToken; 3] { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -98,7 +98,7 @@ fn a_json_schema_sampler_masks_tokens_that_cannot_start_the_schema( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -106,7 +106,7 @@ fn a_json_schema_sampler_masks_tokens_that_cannot_start_the_schema( )] #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -114,7 +114,7 @@ fn a_json_schema_sampler_masks_tokens_that_cannot_start_the_schema( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -122,7 +122,7 @@ fn a_json_schema_sampler_masks_tokens_that_cannot_start_the_schema( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -130,7 +130,7 @@ fn a_json_schema_sampler_masks_tokens_that_cannot_start_the_schema( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -152,7 +152,7 @@ fn a_regex_sampler_masks_tokens_that_cannot_start_a_match( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -160,7 +160,7 @@ fn a_regex_sampler_masks_tokens_that_cannot_start_a_match( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -168,7 +168,7 @@ fn a_regex_sampler_masks_tokens_that_cannot_start_a_match( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -176,7 +176,7 @@ fn a_regex_sampler_masks_tokens_that_cannot_start_a_match( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -198,7 +198,7 @@ fn a_lark_sampler_masks_tokens_that_cannot_start_the_grammar( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -206,7 +206,7 @@ fn a_lark_sampler_masks_tokens_that_cannot_start_the_grammar( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -214,7 +214,7 @@ fn a_lark_sampler_masks_tokens_that_cannot_start_the_grammar( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -222,7 +222,7 @@ fn a_lark_sampler_masks_tokens_that_cannot_start_the_grammar( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -239,7 +239,7 @@ fn an_unknown_grammar_kind_is_reported_as_an_invalid_grammar(fixture: &LlamaFixt #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -247,7 +247,7 @@ fn an_unknown_grammar_kind_is_reported_as_an_invalid_grammar(fixture: &LlamaFixt )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -255,7 +255,7 @@ fn an_unknown_grammar_kind_is_reported_as_an_invalid_grammar(fixture: &LlamaFixt )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -263,7 +263,7 @@ fn an_unknown_grammar_kind_is_reported_as_an_invalid_grammar(fixture: &LlamaFixt )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -280,7 +280,7 @@ fn a_json_schema_that_is_not_json_is_reported_as_an_invalid_grammar(fixture: &Ll #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -288,7 +288,7 @@ fn a_json_schema_that_is_not_json_is_reported_as_an_invalid_grammar(fixture: &Ll )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -296,7 +296,7 @@ fn a_json_schema_that_is_not_json_is_reported_as_an_invalid_grammar(fixture: &Ll )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -304,7 +304,7 @@ fn a_json_schema_that_is_not_json_is_reported_as_an_invalid_grammar(fixture: &Ll )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -322,7 +322,7 @@ fn a_malformed_regex_leaves_llguidance_without_a_parser(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -330,7 +330,7 @@ fn a_malformed_regex_leaves_llguidance_without_a_parser(fixture: &LlamaFixture<' )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -338,7 +338,7 @@ fn a_malformed_regex_leaves_llguidance_without_a_parser(fixture: &LlamaFixture<' )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -346,7 +346,7 @@ fn a_malformed_regex_leaves_llguidance_without_a_parser(fixture: &LlamaFixture<' )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -366,7 +366,7 @@ fn name_callback_returns_llguidance(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -374,7 +374,7 @@ fn name_callback_returns_llguidance(fixture: &LlamaFixture<'_>) -> Result<()> { )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -382,7 +382,7 @@ fn name_callback_returns_llguidance(fixture: &LlamaFixture<'_>) -> Result<()> { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -390,7 +390,7 @@ fn name_callback_returns_llguidance(fixture: &LlamaFixture<'_>) -> Result<()> { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -416,7 +416,7 @@ fn a_sampler_cloned_through_ffi_keeps_its_own_grammar_state( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -424,7 +424,7 @@ fn a_sampler_cloned_through_ffi_keeps_its_own_grammar_state( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -432,7 +432,7 @@ fn a_sampler_cloned_through_ffi_keeps_its_own_grammar_state( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -440,7 +440,7 @@ fn a_sampler_cloned_through_ffi_keeps_its_own_grammar_state( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -481,7 +481,7 @@ fn llguidance_chain_samples_yes_as_the_first_answer_token( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -489,7 +489,7 @@ fn llguidance_chain_samples_yes_as_the_first_answer_token( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -497,7 +497,7 @@ fn llguidance_chain_samples_yes_as_the_first_answer_token( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -505,7 +505,7 @@ fn llguidance_chain_samples_yes_as_the_first_answer_token( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -541,7 +541,7 @@ fn reset_rolls_back_accepted_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -564,7 +564,7 @@ fn accept_maps_an_out_of_vocabulary_token_to_grammar_callback_failure( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -572,7 +572,7 @@ fn accept_maps_an_out_of_vocabulary_token_to_grammar_callback_failure( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -580,7 +580,7 @@ fn accept_maps_an_out_of_vocabulary_token_to_grammar_callback_failure( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -588,7 +588,7 @@ fn accept_maps_an_out_of_vocabulary_token_to_grammar_callback_failure( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -612,7 +612,7 @@ fn samplers_sharing_the_cached_tokenizer_environment_mask_alike( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -663,7 +663,7 @@ fn freshly_loaded(fixture: &LlamaFixture<'_>) -> Result { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -687,7 +687,7 @@ fn creating_a_sampler_reports_a_vocabulary_llama_cpp_cannot_render( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs b/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs index 3f7c9e052..2903f7641 100644 --- a/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs +++ b/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs @@ -8,7 +8,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, diff --git a/llama-cpp-bindings-tests/tests/logits_availability.rs b/llama-cpp-bindings-tests/tests/logits_availability.rs index bd3aff655..28e3de348 100644 --- a/llama-cpp-bindings-tests/tests/logits_availability.rs +++ b/llama-cpp-bindings-tests/tests/logits_availability.rs @@ -12,7 +12,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -39,7 +39,7 @@ fn sampling_after_a_pooled_embeddings_decode_is_refused_instead_of_aborting( #[llama_test( model_source = HuggingFace("LiquidAI/LFM2-ColBERT-350M-GGUF", "LFM2-ColBERT-350M-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -59,7 +59,7 @@ fn a_model_whose_embeddings_graph_has_no_logits_head_produces_no_logits( #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -88,7 +88,7 @@ fn every_logits_accessor_refuses_a_pooled_embeddings_decode(fixture: &LlamaFixtu #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/logits_retrieval.rs b/llama-cpp-bindings-tests/tests/logits_retrieval.rs index 22ec950cb..f78d74ae0 100644 --- a/llama-cpp-bindings-tests/tests/logits_retrieval.rs +++ b/llama-cpp-bindings-tests/tests/logits_retrieval.rs @@ -12,7 +12,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -41,7 +41,7 @@ fn a_logits_row_holds_one_logit_per_vocabulary_token(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -65,7 +65,7 @@ fn get_logits_ith_returns_token_not_initialized_for_unknown_index( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, diff --git a/llama-cpp-bindings-tests/tests/lora_adapters.rs b/llama-cpp-bindings-tests/tests/lora_adapters.rs index 7e5cf2900..7ccf4e72c 100644 --- a/llama-cpp-bindings-tests/tests/lora_adapters.rs +++ b/llama-cpp-bindings-tests/tests/lora_adapters.rs @@ -22,7 +22,7 @@ use llama_cpp_test_harness_macros::llama_test; "Qwen/Qwen2.5-7B-Instruct-GGUF", "qwen2.5-7b-instruct-q2_k.gguf" ), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -53,7 +53,7 @@ fn set_lora_adapters_applies_a_real_adapter(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -75,7 +75,7 @@ fn set_lora_adapters_accepts_an_empty_adapter_set(fixture: &LlamaFixture<'_>) -> "Qwen/Qwen2.5-7B-Instruct-GGUF", "qwen2.5-7b-instruct-q2_k.gguf" ), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -106,7 +106,7 @@ fn setting_lora_adapters_reports_a_status_llama_cpp_returns( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -124,7 +124,7 @@ fn lora_adapter_init_with_a_missing_file_reports_it_not_found(fixture: &LlamaFix #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -145,7 +145,7 @@ fn lora_adapter_init_with_an_invalid_gguf_reports_it_unloadable( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -163,7 +163,7 @@ fn lora_adapter_init_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/memory_availability.rs b/llama-cpp-bindings-tests/tests/memory_availability.rs index b643162ea..0c55d49c2 100644 --- a/llama-cpp-bindings-tests/tests/memory_availability.rs +++ b/llama-cpp-bindings-tests/tests/memory_availability.rs @@ -13,7 +13,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -28,7 +28,7 @@ fn an_encoder_context_keeps_no_memory(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -43,7 +43,7 @@ fn a_causal_embedding_context_keeps_memory(fixture: &LlamaFixture<'_>) -> Result #[llama_test( model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -66,7 +66,7 @@ fn memory_edits_without_a_memory_module_trivially_succeed( #[llama_test( model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -102,7 +102,7 @@ fn memory_operations_needing_a_memory_module_report_its_absence( #[llama_test( model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, diff --git a/llama-cpp-bindings-tests/tests/memory_occupancy.rs b/llama-cpp-bindings-tests/tests/memory_occupancy.rs index 330255a32..469e14ed5 100644 --- a/llama-cpp-bindings-tests/tests/memory_occupancy.rs +++ b/llama-cpp-bindings-tests/tests/memory_occupancy.rs @@ -12,7 +12,7 @@ const PRIMED_TEXT: &str = "Hello world"; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -20,7 +20,7 @@ const PRIMED_TEXT: &str = "Hello world"; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -28,7 +28,7 @@ const PRIMED_TEXT: &str = "Hello world"; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -36,7 +36,7 @@ const PRIMED_TEXT: &str = "Hello world"; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -55,7 +55,7 @@ fn memory_clear_resets_positions(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -63,7 +63,7 @@ fn memory_clear_resets_positions(fixture: &LlamaFixture<'_>) -> Result<()> { )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -71,7 +71,7 @@ fn memory_clear_resets_positions(fixture: &LlamaFixture<'_>) -> Result<()> { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -79,7 +79,7 @@ fn memory_clear_resets_positions(fixture: &LlamaFixture<'_>) -> Result<()> { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -106,7 +106,7 @@ fn memory_seq_pos_max_is_the_position_of_the_last_decoded_token( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -114,7 +114,7 @@ fn memory_seq_pos_max_is_the_position_of_the_last_decoded_token( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -122,7 +122,7 @@ fn memory_seq_pos_max_is_the_position_of_the_last_decoded_token( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -130,7 +130,7 @@ fn memory_seq_pos_max_is_the_position_of_the_last_decoded_token( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/memory_position_bounds.rs b/llama-cpp-bindings-tests/tests/memory_position_bounds.rs index 28459d42a..511b0a1f3 100644 --- a/llama-cpp-bindings-tests/tests/memory_position_bounds.rs +++ b/llama-cpp-bindings-tests/tests/memory_position_bounds.rs @@ -13,7 +13,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -21,7 +21,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -29,7 +29,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -37,7 +37,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -60,7 +60,7 @@ fn memory_seq_cp_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Res #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -68,7 +68,7 @@ fn memory_seq_cp_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Res )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -76,7 +76,7 @@ fn memory_seq_cp_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Res )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -84,7 +84,7 @@ fn memory_seq_cp_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Res )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -107,7 +107,7 @@ fn memory_seq_cp_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Res #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -115,7 +115,7 @@ fn memory_seq_cp_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Res )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -123,7 +123,7 @@ fn memory_seq_cp_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Res )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -131,7 +131,7 @@ fn memory_seq_cp_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Res )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -154,7 +154,7 @@ fn memory_seq_rm_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Res #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -162,7 +162,7 @@ fn memory_seq_rm_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Res )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -170,7 +170,7 @@ fn memory_seq_rm_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Res )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -178,7 +178,7 @@ fn memory_seq_rm_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Res )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -201,7 +201,7 @@ fn memory_seq_rm_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Res #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -209,7 +209,7 @@ fn memory_seq_rm_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Res )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -217,7 +217,7 @@ fn memory_seq_rm_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Res )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -225,7 +225,7 @@ fn memory_seq_rm_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Res )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -248,7 +248,7 @@ fn memory_seq_add_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Re #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -256,7 +256,7 @@ fn memory_seq_add_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Re )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -264,7 +264,7 @@ fn memory_seq_add_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Re )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -272,7 +272,7 @@ fn memory_seq_add_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Re )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -295,7 +295,7 @@ fn memory_seq_add_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Re #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -303,7 +303,7 @@ fn memory_seq_add_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Re )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -311,7 +311,7 @@ fn memory_seq_add_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Re )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -319,7 +319,7 @@ fn memory_seq_add_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Re )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -343,7 +343,7 @@ fn memory_seq_div_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Re #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -351,7 +351,7 @@ fn memory_seq_div_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Re )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -359,7 +359,7 @@ fn memory_seq_div_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Re )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -367,7 +367,7 @@ fn memory_seq_div_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Re )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs b/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs index 1faa7e308..fc87dd200 100644 --- a/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs +++ b/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs @@ -13,7 +13,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -21,7 +21,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -41,7 +41,7 @@ fn memory_seq_add_refuses_an_mrope_model(fixture: &LlamaFixture<'_>) -> Result<( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -49,7 +49,7 @@ fn memory_seq_add_refuses_an_mrope_model(fixture: &LlamaFixture<'_>) -> Result<( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -70,7 +70,7 @@ fn memory_seq_div_refuses_an_mrope_model(fixture: &LlamaFixture<'_>) -> Result<( #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -94,7 +94,7 @@ fn memory_seq_add_shifts_the_positions_of_a_standard_rope_model( #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -122,7 +122,7 @@ fn memory_seq_div_divides_the_positions_of_a_standard_rope_model( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -146,7 +146,7 @@ fn memory_seq_add_refuses_a_model_whose_rope_type_llama_cpp_does_not_define( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/memory_priming.rs b/llama-cpp-bindings-tests/tests/memory_priming.rs index b38e38a0b..a8d84c4c7 100644 --- a/llama-cpp-bindings-tests/tests/memory_priming.rs +++ b/llama-cpp-bindings-tests/tests/memory_priming.rs @@ -13,7 +13,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 256, @@ -37,7 +37,7 @@ fn priming_memory_surfaces_a_tokenization_error(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 256, @@ -59,7 +59,7 @@ fn priming_memory_surfaces_a_batch_add_error(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 256, diff --git a/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs b/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs index c84a07d45..7adb081ea 100644 --- a/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs +++ b/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs @@ -8,7 +8,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -16,7 +16,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -24,7 +24,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -32,7 +32,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -50,7 +50,7 @@ fn memory_seq_rm_accepts_an_explicit_range(fixture: &LlamaFixture<'_>) -> Result #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -58,7 +58,7 @@ fn memory_seq_rm_accepts_an_explicit_range(fixture: &LlamaFixture<'_>) -> Result )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -66,7 +66,7 @@ fn memory_seq_rm_accepts_an_explicit_range(fixture: &LlamaFixture<'_>) -> Result )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -74,7 +74,7 @@ fn memory_seq_rm_accepts_an_explicit_range(fixture: &LlamaFixture<'_>) -> Result )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -96,7 +96,7 @@ fn memory_seq_cp_copies_a_whole_primed_sequence(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -104,7 +104,7 @@ fn memory_seq_cp_copies_a_whole_primed_sequence(fixture: &LlamaFixture<'_>) -> R )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -112,7 +112,7 @@ fn memory_seq_cp_copies_a_whole_primed_sequence(fixture: &LlamaFixture<'_>) -> R )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -120,7 +120,7 @@ fn memory_seq_cp_copies_a_whole_primed_sequence(fixture: &LlamaFixture<'_>) -> R )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -138,7 +138,7 @@ fn memory_seq_cp_accepts_an_explicit_range(fixture: &LlamaFixture<'_>) -> Result #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs b/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs index d40f25521..dd6204bd1 100644 --- a/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs +++ b/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs @@ -13,7 +13,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -21,7 +21,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -29,7 +29,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -37,7 +37,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -63,7 +63,7 @@ fn memory_seq_pos_max_rejects_out_of_range_sequence(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -71,7 +71,7 @@ fn memory_seq_pos_max_rejects_out_of_range_sequence(fixture: &LlamaFixture<'_>) )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -79,7 +79,7 @@ fn memory_seq_pos_max_rejects_out_of_range_sequence(fixture: &LlamaFixture<'_>) )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -87,7 +87,7 @@ fn memory_seq_pos_max_rejects_out_of_range_sequence(fixture: &LlamaFixture<'_>) )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -110,7 +110,7 @@ fn memory_seq_rm_rejects_sequence_id_exceeding_i32_max(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -136,7 +136,7 @@ fn memory_seq_cp_rejects_a_destination_outside_the_context_sequences( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -160,7 +160,7 @@ fn memory_seq_keep_rejects_a_sequence_outside_the_context_sequences( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -184,7 +184,7 @@ fn memory_seq_add_rejects_a_negative_sequence_id(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -210,7 +210,7 @@ fn memory_seq_div_rejects_a_sequence_outside_the_context_sequences( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/model_architecture.rs b/llama-cpp-bindings-tests/tests/model_architecture.rs index 775a9c662..a16739b82 100644 --- a/llama-cpp-bindings-tests/tests/model_architecture.rs +++ b/llama-cpp-bindings-tests/tests/model_architecture.rs @@ -4,7 +4,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -12,7 +12,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -20,7 +20,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -28,7 +28,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -40,7 +40,7 @@ fn is_recurrent_returns_false_for_transformer(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -48,7 +48,7 @@ fn is_recurrent_returns_false_for_transformer(fixture: &LlamaFixture<'_>) { )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -63,7 +63,7 @@ fn is_hybrid_returns_false_for_non_hybrid_default_models(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -71,7 +71,7 @@ fn is_hybrid_returns_false_for_non_hybrid_default_models(fixture: &LlamaFixture< )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -86,7 +86,7 @@ fn is_hybrid_returns_true_for_qwen_default_models(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -98,7 +98,7 @@ fn rope_type_reports_normal_rope_for_a_llama_architecture_model(fixture: &LlamaF #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -110,7 +110,7 @@ fn rope_type_reports_normal_rope_for_glm47_flash(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -118,7 +118,7 @@ fn rope_type_reports_normal_rope_for_glm47_flash(fixture: &LlamaFixture<'_>) { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/model_derived_data_caching.rs b/llama-cpp-bindings-tests/tests/model_derived_data_caching.rs index b4a8d9add..7fdc0fccf 100644 --- a/llama-cpp-bindings-tests/tests/model_derived_data_caching.rs +++ b/llama-cpp-bindings-tests/tests/model_derived_data_caching.rs @@ -8,7 +8,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, @@ -25,7 +25,7 @@ fn approximate_tok_env_is_cached_across_calls(fixture: &LlamaFixture<'_>) -> Res #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/model_hyperparameters.rs b/llama-cpp-bindings-tests/tests/model_hyperparameters.rs index 8e9b23da0..1535003f9 100644 --- a/llama-cpp-bindings-tests/tests/model_hyperparameters.rs +++ b/llama-cpp-bindings-tests/tests/model_hyperparameters.rs @@ -5,7 +5,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -13,7 +13,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -21,7 +21,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -29,7 +29,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/model_loading_errors.rs b/llama-cpp-bindings-tests/tests/model_loading_errors.rs index 49178aacd..c706ed7d0 100644 --- a/llama-cpp-bindings-tests/tests/model_loading_errors.rs +++ b/llama-cpp-bindings-tests/tests/model_loading_errors.rs @@ -16,7 +16,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -37,7 +37,7 @@ fn load_model_with_a_missing_file_reports_it_not_found(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -59,7 +59,7 @@ fn load_model_with_invalid_file_content_reports_it_unloadable( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -81,7 +81,7 @@ fn load_model_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/model_metadata.rs b/llama-cpp-bindings-tests/tests/model_metadata.rs index 8f3ae13f3..790f051b0 100644 --- a/llama-cpp-bindings-tests/tests/model_metadata.rs +++ b/llama-cpp-bindings-tests/tests/model_metadata.rs @@ -11,7 +11,7 @@ const OUT_OF_RANGE_META_INDEX_RETURN: i32 = -1; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -19,7 +19,7 @@ const OUT_OF_RANGE_META_INDEX_RETURN: i32 = -1; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -27,7 +27,7 @@ const OUT_OF_RANGE_META_INDEX_RETURN: i32 = -1; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -35,7 +35,7 @@ const OUT_OF_RANGE_META_INDEX_RETURN: i32 = -1; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -47,7 +47,7 @@ fn meta_count_returns_positive(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -55,7 +55,7 @@ fn meta_count_returns_positive(fixture: &LlamaFixture<'_>) { )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -63,7 +63,7 @@ fn meta_count_returns_positive(fixture: &LlamaFixture<'_>) { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -71,7 +71,7 @@ fn meta_count_returns_positive(fixture: &LlamaFixture<'_>) { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -85,7 +85,7 @@ fn the_first_metadata_entry_has_a_nonempty_key(fixture: &LlamaFixture<'_>) -> Re #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -93,7 +93,7 @@ fn the_first_metadata_entry_has_a_nonempty_key(fixture: &LlamaFixture<'_>) -> Re )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -101,7 +101,7 @@ fn the_first_metadata_entry_has_a_nonempty_key(fixture: &LlamaFixture<'_>) -> Re )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -109,7 +109,7 @@ fn the_first_metadata_entry_has_a_nonempty_key(fixture: &LlamaFixture<'_>) -> Re )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -123,7 +123,7 @@ fn the_first_metadata_entry_has_a_nonempty_value(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -131,7 +131,7 @@ fn the_first_metadata_entry_has_a_nonempty_value(fixture: &LlamaFixture<'_>) -> )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -139,7 +139,7 @@ fn the_first_metadata_entry_has_a_nonempty_value(fixture: &LlamaFixture<'_>) -> )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -147,7 +147,7 @@ fn the_first_metadata_entry_has_a_nonempty_value(fixture: &LlamaFixture<'_>) -> )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -164,7 +164,7 @@ fn meta_key_by_index_out_of_range_reports_the_negative_llama_cpp_return( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -172,7 +172,7 @@ fn meta_key_by_index_out_of_range_reports_the_negative_llama_cpp_return( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -180,7 +180,7 @@ fn meta_key_by_index_out_of_range_reports_the_negative_llama_cpp_return( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -188,7 +188,7 @@ fn meta_key_by_index_out_of_range_reports_the_negative_llama_cpp_return( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -205,7 +205,7 @@ fn meta_val_str_by_index_out_of_range_reports_the_negative_llama_cpp_return( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -213,7 +213,7 @@ fn meta_val_str_by_index_out_of_range_reports_the_negative_llama_cpp_return( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -221,7 +221,7 @@ fn meta_val_str_by_index_out_of_range_reports_the_negative_llama_cpp_return( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -229,7 +229,7 @@ fn meta_val_str_by_index_out_of_range_reports_the_negative_llama_cpp_return( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -245,7 +245,7 @@ fn meta_val_str_returns_value_for_known_key(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -253,7 +253,7 @@ fn meta_val_str_returns_value_for_known_key(fixture: &LlamaFixture<'_>) -> Resul )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -261,7 +261,7 @@ fn meta_val_str_returns_value_for_known_key(fixture: &LlamaFixture<'_>) -> Resul )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -269,7 +269,7 @@ fn meta_val_str_returns_value_for_known_key(fixture: &LlamaFixture<'_>) -> Resul )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -288,7 +288,7 @@ fn meta_val_str_reads_a_value_longer_than_its_initial_buffer_whole( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -296,7 +296,7 @@ fn meta_val_str_reads_a_value_longer_than_its_initial_buffer_whole( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -304,7 +304,7 @@ fn meta_val_str_reads_a_value_longer_than_its_initial_buffer_whole( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -312,7 +312,7 @@ fn meta_val_str_reads_a_value_longer_than_its_initial_buffer_whole( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/multimodal_audio.rs b/llama-cpp-bindings-tests/tests/multimodal_audio.rs index bb48f39b2..2db5f411c 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_audio.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_audio.rs @@ -151,7 +151,7 @@ fn assert_audio_transcription( "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", "Llama-3.2-1B-Instruct-Q4_K_M.gguf" ), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4096, n_batch = 512, @@ -176,7 +176,7 @@ fn ultravox_transcribes_the_spoken_sentence_after_a_preamble( #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4096, n_batch = 512, @@ -193,7 +193,7 @@ fn gemma4_transcribes_the_quick_brown_fox_sentence(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4096, n_batch = 512, @@ -213,7 +213,7 @@ fn gemma4_transcribes_the_orange_cat_sentence(fixture: &LlamaFixture<'_>) -> Res "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", "Llama-3.2-1B-Instruct-Q4_K_M.gguf" ), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4096, n_batch = 512, @@ -225,7 +225,7 @@ fn gemma4_transcribes_the_orange_cat_sentence(fixture: &LlamaFixture<'_>) -> Res )] #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4096, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs b/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs index c43d04815..dd5155f05 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs @@ -14,7 +14,7 @@ const LLAMAS_JPG_HEIGHT: u32 = 427; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -23,7 +23,7 @@ const LLAMAS_JPG_HEIGHT: u32 = 427; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -51,7 +51,7 @@ fn from_buffer_decodes_image_bytes_into_an_image_bitmap_of_their_size( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -60,7 +60,7 @@ fn from_buffer_decodes_image_bytes_into_an_image_bitmap_of_their_size( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs index 9ac29fef7..8beb3de5e 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs @@ -8,7 +8,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -17,7 +17,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs index 2c893054e..fad59e28e 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs @@ -9,7 +9,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -18,7 +18,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs index 05f6d1f38..08658cbcf 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs @@ -144,7 +144,7 @@ fn tokenize_question_about_llamas( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 64, n_batch = 64, @@ -153,7 +153,7 @@ fn tokenize_question_about_llamas( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 64, n_batch = 64, @@ -190,7 +190,7 @@ fn eval_chunks_returns_batch_size_exceeds_context_limit_for_huge_batch( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -221,7 +221,7 @@ fn eval_chunks_reports_an_exception_llama_cpp_throws_while_evaluating_a_chunk( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -259,7 +259,7 @@ fn eval_chunks_reports_a_chunk_llama_cpp_does_not_return_before_evaluating_any( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -268,7 +268,7 @@ fn eval_chunks_reports_a_chunk_llama_cpp_does_not_return_before_evaluating_any( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -297,7 +297,7 @@ fn eval_chunks_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -306,7 +306,7 @@ fn eval_chunks_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_> )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -333,7 +333,7 @@ fn fitting_no_chunks_leaves_nothing_to_evaluate(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -342,7 +342,7 @@ fn fitting_no_chunks_leaves_nothing_to_evaluate(fixture: &LlamaFixture<'_>) -> R )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -377,7 +377,7 @@ fn eval_single_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -415,7 +415,7 @@ fn eval_single_rejects_a_batch_larger_than_the_context_batch_before_evaluating( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -448,7 +448,7 @@ fn eval_chunks_rejects_a_sequence_the_context_does_not_have_instead_of_aborting( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -457,7 +457,7 @@ fn eval_chunks_rejects_a_sequence_the_context_does_not_have_instead_of_aborting( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -506,7 +506,7 @@ fn eval_single_rejects_a_micro_batch_smaller_than_the_fitted_one_before_evaluati #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -515,7 +515,7 @@ fn eval_single_rejects_a_micro_batch_smaller_than_the_fitted_one_before_evaluati )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -558,7 +558,7 @@ fn classifier_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 8192, n_batch = 512, @@ -567,7 +567,7 @@ fn classifier_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_>) )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 8192, n_batch = 512, @@ -619,7 +619,7 @@ fn eval_chunks_advances_past_every_position_of_a_standard_image( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 8192, n_batch = 512, @@ -628,7 +628,7 @@ fn eval_chunks_advances_past_every_position_of_a_standard_image( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 8192, n_batch = 512, @@ -648,7 +648,7 @@ fn eval_chunks_evaluates_images_of_ordinary_and_extreme_dimensions( #[llama_test( model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 2048, @@ -696,7 +696,7 @@ fn gemma3_decoding_follows_the_causal_attention_the_mtmd_helper_leaves_on_the_co #[llama_test( model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 2048, @@ -736,7 +736,7 @@ fn gemma3_eval_chunks_rejects_a_non_causal_image_larger_than_the_micro_batch_bef #[llama_test( model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 2048, @@ -775,7 +775,7 @@ fn gemma3_fitting_rejects_a_non_causal_image_larger_than_the_micro_batch( #[llama_test( model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 2048, @@ -826,7 +826,7 @@ fn gemma3_classifier_leaves_the_prompt_untouched_when_a_non_causal_image_exceeds #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 256, @@ -879,7 +879,7 @@ fn classifier_reports_an_image_the_context_has_no_room_left_for( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 256, @@ -909,7 +909,7 @@ fn eval_chunks_reports_an_image_the_context_has_no_room_left_for( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 256, @@ -938,7 +938,7 @@ fn fitting_reports_a_chunk_llama_cpp_does_not_return(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 256, @@ -967,7 +967,7 @@ fn fitting_reports_a_chunk_type_the_bindings_do_not_recognize( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 256, diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs index c5b7a1887..c5b944f0b 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs @@ -26,7 +26,7 @@ fn first_image_chunk(chunks: &MtmdInputChunks) -> Result { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -35,7 +35,7 @@ fn first_image_chunk(chunks: &MtmdInputChunks) -> Result { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -51,7 +51,7 @@ fn text_chunk_has_text_type(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -60,7 +60,7 @@ fn text_chunk_has_text_type(fixture: &LlamaFixture<'_>) -> Result<()> { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -81,7 +81,7 @@ fn text_chunk_returns_text_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -90,7 +90,7 @@ fn text_chunk_returns_text_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -106,7 +106,7 @@ fn chunk_n_tokens_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -115,7 +115,7 @@ fn chunk_n_tokens_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -131,7 +131,7 @@ fn chunk_n_positions_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -140,7 +140,7 @@ fn chunk_n_positions_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -157,7 +157,7 @@ fn text_chunk_id_returns_none(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -166,7 +166,7 @@ fn text_chunk_id_returns_none(fixture: &LlamaFixture<'_>) -> Result<()> { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -186,7 +186,7 @@ fn an_image_chunk_has_no_text_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -195,7 +195,7 @@ fn an_image_chunk_has_no_text_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -214,7 +214,7 @@ fn an_image_chunk_carries_the_empty_id_of_a_bitmap_without_one( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -223,7 +223,7 @@ fn an_image_chunk_carries_the_empty_id_of_a_bitmap_without_one( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -250,7 +250,7 @@ fn tokenizing_text_with_an_image_produces_chunks(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -259,7 +259,7 @@ fn tokenizing_text_with_an_image_produces_chunks(fixture: &LlamaFixture<'_>) -> )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -289,7 +289,7 @@ fn tokenizing_more_bitmaps_than_media_markers_reports_a_tokenization_failure( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -298,7 +298,7 @@ fn tokenizing_more_bitmaps_than_media_markers_reports_a_tokenization_failure( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -325,7 +325,7 @@ fn tokenize_refuses_text_with_a_nul_byte(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs index b77a8617b..d402d824d 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs @@ -57,7 +57,7 @@ fn load_fixture_bitmap(mtmd_ctx: &MtmdContext, file_name: &str) -> Result) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -65,7 +65,7 @@ fn qwen_image_chunks_decode_causally(fixture: &LlamaFixture<'_>) -> Result<()> { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -85,7 +85,7 @@ fn qwen_mmproj_reports_vision_only_chat_capabilities(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -94,7 +94,7 @@ fn qwen_mmproj_reports_vision_only_chat_capabilities(fixture: &LlamaFixture<'_>) )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -113,7 +113,7 @@ fn decode_use_mrope_is_true_for_qwen_vision(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -122,7 +122,7 @@ fn decode_use_mrope_is_true_for_qwen_vision(fixture: &LlamaFixture<'_>) { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -144,7 +144,7 @@ fn get_audio_sample_rate_is_none_for_vision_only_mmproj(fixture: &LlamaFixture<' "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", "Llama-3.2-1B-Instruct-Q4_K_M.gguf" ), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4096, n_batch = 512, @@ -156,7 +156,7 @@ fn get_audio_sample_rate_is_none_for_vision_only_mmproj(fixture: &LlamaFixture<' )] #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4096, n_batch = 512, @@ -179,7 +179,7 @@ fn audio_mmproj_reports_audio_support_at_sixteen_kilohertz(fixture: &LlamaFixtur #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4096, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs b/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs index 26aed74ec..e9c43dbd5 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs @@ -11,7 +11,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -20,7 +20,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -40,7 +40,7 @@ fn init_from_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs b/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs index ff50e79ba..0d850a930 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs @@ -46,7 +46,7 @@ fn tokens_of_text_chunks(chunks: &MtmdInputChunks) -> Result> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -89,7 +89,7 @@ fn only_bitmap_parts_become_image_chunks(fixture: &LlamaFixture<'_>) -> Result<( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -117,7 +117,7 @@ fn a_media_marker_inside_a_text_part_tokenizes_as_plain_text( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -158,7 +158,7 @@ fn parse_special_applies_to_each_text_part_on_its_own(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -183,7 +183,7 @@ fn a_text_part_containing_a_nul_byte_is_rejected(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -211,7 +211,7 @@ fn tokenizing_parts_reports_a_chunk_list_llama_cpp_cannot_allocate(fixture: &Lla #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs index 5cb98c39b..3e0fc4e9c 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs @@ -25,7 +25,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 8192, n_batch = 512, @@ -116,7 +116,7 @@ fn gemma4_classifier_emits_reasoning_for_multimodal_thinking_prompt( #[llama_test( model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4096, n_batch = 512, @@ -226,7 +226,7 @@ fn mistral3_classifier_emits_reasoning_for_multimodal_thinking_prompt( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4096, n_batch = 512, @@ -235,7 +235,7 @@ fn mistral3_classifier_emits_reasoning_for_multimodal_thinking_prompt( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4096, n_batch = 512, @@ -244,7 +244,7 @@ fn mistral3_classifier_emits_reasoning_for_multimodal_thinking_prompt( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 8192, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs b/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs index eea291302..1414447bc 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs @@ -68,7 +68,7 @@ fn build_multimodal_chunks_and_eval_into_usage( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4096, n_batch = 512, @@ -94,7 +94,7 @@ fn prompt_tokens_match_text_chunk_total(fixture: &LlamaFixture<'_>) -> Result<() #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4096, n_batch = 512, @@ -120,7 +120,7 @@ fn input_image_tokens_match_image_chunk_total(fixture: &LlamaFixture<'_>) -> Res #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4096, n_batch = 512, @@ -151,7 +151,7 @@ fn input_audio_tokens_are_zero_for_image_only_input(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4096, n_batch = 512, @@ -176,7 +176,7 @@ fn completion_tokens_are_zero_after_eval_before_generation( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -241,7 +241,7 @@ fn text_chunk_records_prompt_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, diff --git a/llama-cpp-bindings-tests/tests/parameter_fitting.rs b/llama-cpp-bindings-tests/tests/parameter_fitting.rs index 66de725f0..db79904a9 100644 --- a/llama-cpp-bindings-tests/tests/parameter_fitting.rs +++ b/llama-cpp-bindings-tests/tests/parameter_fitting.rs @@ -16,7 +16,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -24,7 +24,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -32,7 +32,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -40,7 +40,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -70,7 +70,7 @@ fn fit_params_fits_a_nonzero_context_for_the_model(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -122,7 +122,7 @@ fn fit_params_aligns_the_extra_model_context_with_the_fitted_one( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs b/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs index e93c7d264..d65830f1b 100644 --- a/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs +++ b/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs @@ -12,7 +12,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -20,7 +20,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -28,7 +28,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -36,7 +36,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -75,7 +75,7 @@ fn feed_prompt_to_batch_increments_pending_prompt_tokens(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -83,7 +83,7 @@ fn feed_prompt_to_batch_increments_pending_prompt_tokens(fixture: &LlamaFixture< )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -91,7 +91,7 @@ fn feed_prompt_to_batch_increments_pending_prompt_tokens(fixture: &LlamaFixture< )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -99,7 +99,7 @@ fn feed_prompt_to_batch_increments_pending_prompt_tokens(fixture: &LlamaFixture< )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -130,7 +130,7 @@ fn feed_prompt_sequence_to_batch_stages_all_tokens(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -138,7 +138,7 @@ fn feed_prompt_sequence_to_batch_stages_all_tokens(fixture: &LlamaFixture<'_>) - )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -146,7 +146,7 @@ fn feed_prompt_sequence_to_batch_stages_all_tokens(fixture: &LlamaFixture<'_>) - )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -154,7 +154,7 @@ fn feed_prompt_sequence_to_batch_stages_all_tokens(fixture: &LlamaFixture<'_>) - )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -198,7 +198,7 @@ fn commit_prompt_tokens_promotes_pending_count_to_usage_and_clears( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -206,7 +206,7 @@ fn commit_prompt_tokens_promotes_pending_count_to_usage_and_clears( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -214,7 +214,7 @@ fn commit_prompt_tokens_promotes_pending_count_to_usage_and_clears( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -222,7 +222,7 @@ fn commit_prompt_tokens_promotes_pending_count_to_usage_and_clears( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -259,7 +259,7 @@ fn discard_pending_prompt_tokens_clears_count_without_recording_usage( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -281,7 +281,7 @@ fn cached_prompt_tokens_cannot_exceed_the_recorded_prompt(fixture: &LlamaFixture #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -305,7 +305,7 @@ fn a_prompt_token_refused_by_a_full_batch_is_not_counted(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/reasoning_classification.rs b/llama-cpp-bindings-tests/tests/reasoning_classification.rs index f22a2edc8..0c4723785 100644 --- a/llama-cpp-bindings-tests/tests/reasoning_classification.rs +++ b/llama-cpp-bindings-tests/tests/reasoning_classification.rs @@ -19,7 +19,7 @@ const MAX_REASONING_ANSWER_TOKENS: i32 = 512; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 8192, n_batch = 2048, @@ -44,7 +44,7 @@ fn deepseek_r1_8b_answers_without_reasoning_after_its_reasoning_is_closed( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 8192, n_batch = 2048, @@ -69,7 +69,7 @@ fn deepseek_r1_8b_reasoning_streams_match_the_parsed_answer_when_thinking_is_ena #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 8192, n_batch = 2048, @@ -94,7 +94,7 @@ fn gemma4_answers_without_reasoning_when_thinking_is_disabled( #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 8192, n_batch = 2048, @@ -135,7 +135,7 @@ fn gemma4_reasoning_streams_match_the_parsed_answer_when_reasoning_is_prefilled( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 8192, n_batch = 2048, @@ -160,7 +160,7 @@ fn glm47_answers_without_reasoning_when_thinking_is_disabled( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 8192, n_batch = 2048, @@ -185,7 +185,7 @@ fn glm47_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( #[llama_test( model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 8192, n_batch = 2048, @@ -210,7 +210,7 @@ fn mistral3_answers_without_reasoning_after_its_reasoning_is_closed( #[llama_test( model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 8192, n_batch = 2048, @@ -235,7 +235,7 @@ fn mistral3_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 8192, n_batch = 2048, @@ -260,7 +260,7 @@ fn qwen35_answers_without_reasoning_when_thinking_is_disabled( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 8192, n_batch = 2048, @@ -285,7 +285,7 @@ fn qwen35_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 8192, n_batch = 2048, @@ -310,7 +310,7 @@ fn qwen36_answers_without_reasoning_when_thinking_is_disabled( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 8192, n_batch = 2048, diff --git a/llama-cpp-bindings-tests/tests/reranking.rs b/llama-cpp-bindings-tests/tests/reranking.rs index e03e8e440..730b025b3 100644 --- a/llama-cpp-bindings-tests/tests/reranking.rs +++ b/llama-cpp-bindings-tests/tests/reranking.rs @@ -23,7 +23,7 @@ fn cosine_similarity(left_embedding: &[f32], right_embedding: &[f32]) -> f32 { #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, diff --git a/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs b/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs index 1e28edafb..e1dec307f 100644 --- a/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs +++ b/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs @@ -27,7 +27,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -35,7 +35,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -43,7 +43,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -51,7 +51,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -67,7 +67,7 @@ fn chat_session_classifier_starts_in_the_pending_section(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -75,7 +75,7 @@ fn chat_session_classifier_starts_in_the_pending_section(fixture: &LlamaFixture< )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -83,7 +83,7 @@ fn chat_session_classifier_starts_in_the_pending_section(fixture: &LlamaFixture< )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -91,7 +91,7 @@ fn chat_session_classifier_starts_in_the_pending_section(fixture: &LlamaFixture< )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -125,7 +125,7 @@ fn undeterminable_classifier_releases_each_token_with_its_visible_and_raw_piece( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -158,7 +158,7 @@ fn finishing_a_chat_answer_settles_its_pending_tokens(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -166,7 +166,7 @@ fn finishing_a_chat_answer_settles_its_pending_tokens(fixture: &LlamaFixture<'_> )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -174,7 +174,7 @@ fn finishing_a_chat_answer_settles_its_pending_tokens(fixture: &LlamaFixture<'_> )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -182,7 +182,7 @@ fn finishing_a_chat_answer_settles_its_pending_tokens(fixture: &LlamaFixture<'_> )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -216,7 +216,7 @@ fn ingest_counts_every_token_through_the_end_of_generation( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -257,7 +257,7 @@ fn failed_classification() -> IngestError { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -284,7 +284,7 @@ fn ingesting_a_generated_token_reports_a_classification_llama_cpp_fails( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -312,7 +312,7 @@ fn ending_the_generation_reports_a_classification_llama_cpp_fails( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs b/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs index da73b05f6..49c5c297c 100644 --- a/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs +++ b/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs @@ -7,7 +7,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -33,7 +33,7 @@ fn a_greedy_chain_accepts_many_borrowed_tokens(fixture: &LlamaFixture<'_>) -> Re #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, diff --git a/llama-cpp-bindings-tests/tests/session_file_path_validation.rs b/llama-cpp-bindings-tests/tests/session_file_path_validation.rs index 2fbcac9b2..5a3e72bb3 100644 --- a/llama-cpp-bindings-tests/tests/session_file_path_validation.rs +++ b/llama-cpp-bindings-tests/tests/session_file_path_validation.rs @@ -14,7 +14,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -22,7 +22,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -30,7 +30,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -38,7 +38,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -61,7 +61,7 @@ fn state_save_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -69,7 +69,7 @@ fn state_save_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) - )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -77,7 +77,7 @@ fn state_save_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) - )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -85,7 +85,7 @@ fn state_save_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) - )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -108,7 +108,7 @@ fn state_load_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -116,7 +116,7 @@ fn state_load_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) - )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -124,7 +124,7 @@ fn state_load_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) - )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -132,7 +132,7 @@ fn state_load_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) - )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -155,7 +155,7 @@ fn state_seq_save_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -163,7 +163,7 @@ fn state_seq_save_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_ )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -171,7 +171,7 @@ fn state_seq_save_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_ )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -179,7 +179,7 @@ fn state_seq_save_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_ )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -202,7 +202,7 @@ fn state_seq_load_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -210,7 +210,7 @@ fn state_seq_load_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_ )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -218,7 +218,7 @@ fn state_seq_load_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_ )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -226,7 +226,7 @@ fn state_seq_load_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_ )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -249,7 +249,7 @@ fn state_save_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -257,7 +257,7 @@ fn state_save_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) -> )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -265,7 +265,7 @@ fn state_save_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) -> )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -273,7 +273,7 @@ fn state_save_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) -> )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -296,7 +296,7 @@ fn state_load_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -304,7 +304,7 @@ fn state_load_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) -> )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -312,7 +312,7 @@ fn state_load_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) -> )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -320,7 +320,7 @@ fn state_load_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) -> )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -343,7 +343,7 @@ fn state_seq_save_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -351,7 +351,7 @@ fn state_seq_save_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_> )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -359,7 +359,7 @@ fn state_seq_save_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_> )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -367,7 +367,7 @@ fn state_seq_save_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_> )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs b/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs index d9814f73b..fdcd9a664 100644 --- a/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs +++ b/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs @@ -14,7 +14,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -22,7 +22,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -30,7 +30,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -38,7 +38,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -72,7 +72,7 @@ fn state_load_file_with_too_small_a_token_capacity_reports_failed_to_load( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -80,7 +80,7 @@ fn state_load_file_with_too_small_a_token_capacity_reports_failed_to_load( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -88,7 +88,7 @@ fn state_load_file_with_too_small_a_token_capacity_reports_failed_to_load( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -96,7 +96,7 @@ fn state_load_file_with_too_small_a_token_capacity_reports_failed_to_load( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/session_files.rs b/llama-cpp-bindings-tests/tests/session_files.rs index 0ae26b1f5..e18c961e6 100644 --- a/llama-cpp-bindings-tests/tests/session_files.rs +++ b/llama-cpp-bindings-tests/tests/session_files.rs @@ -16,7 +16,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -24,7 +24,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -32,7 +32,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -40,7 +40,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -72,7 +72,7 @@ fn a_loaded_state_file_restores_the_tokens_it_was_saved_with( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -80,7 +80,7 @@ fn a_loaded_state_file_restores_the_tokens_it_was_saved_with( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -88,7 +88,7 @@ fn a_loaded_state_file_restores_the_tokens_it_was_saved_with( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -96,7 +96,7 @@ fn a_loaded_state_file_restores_the_tokens_it_was_saved_with( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -134,7 +134,7 @@ fn a_loaded_sequence_state_file_restores_its_tokens_and_bytes( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -142,7 +142,7 @@ fn a_loaded_sequence_state_file_restores_its_tokens_and_bytes( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -150,7 +150,7 @@ fn a_loaded_sequence_state_file_restores_its_tokens_and_bytes( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -158,7 +158,7 @@ fn a_loaded_sequence_state_file_restores_its_tokens_and_bytes( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -181,7 +181,7 @@ fn state_load_file_reports_a_missing_file_as_failed_to_load( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -189,7 +189,7 @@ fn state_load_file_reports_a_missing_file_as_failed_to_load( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -197,7 +197,7 @@ fn state_load_file_reports_a_missing_file_as_failed_to_load( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -205,7 +205,7 @@ fn state_load_file_reports_a_missing_file_as_failed_to_load( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -228,7 +228,7 @@ fn state_seq_load_file_reports_a_missing_file_as_failed_to_load( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -236,7 +236,7 @@ fn state_seq_load_file_reports_a_missing_file_as_failed_to_load( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -244,7 +244,7 @@ fn state_seq_load_file_reports_a_missing_file_as_failed_to_load( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -252,7 +252,7 @@ fn state_seq_load_file_reports_a_missing_file_as_failed_to_load( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -273,7 +273,7 @@ fn state_save_file_to_invalid_directory_returns_failed_to_save( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -281,7 +281,7 @@ fn state_save_file_to_invalid_directory_returns_failed_to_save( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -289,7 +289,7 @@ fn state_save_file_to_invalid_directory_returns_failed_to_save( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -297,7 +297,7 @@ fn state_save_file_to_invalid_directory_returns_failed_to_save( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/special_tokens.rs b/llama-cpp-bindings-tests/tests/special_tokens.rs index e8ed803a4..a55d2ad41 100644 --- a/llama-cpp-bindings-tests/tests/special_tokens.rs +++ b/llama-cpp-bindings-tests/tests/special_tokens.rs @@ -8,7 +8,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -16,7 +16,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -24,7 +24,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -32,7 +32,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -48,7 +48,7 @@ fn the_eos_token_differs_from_the_bos_token(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -61,7 +61,7 @@ fn model_without_a_bos_token_reports_none(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -69,7 +69,7 @@ fn model_without_a_bos_token_reports_none(fixture: &LlamaFixture<'_>) { )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -77,7 +77,7 @@ fn model_without_a_bos_token_reports_none(fixture: &LlamaFixture<'_>) { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -85,7 +85,7 @@ fn model_without_a_bos_token_reports_none(fixture: &LlamaFixture<'_>) { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -101,7 +101,7 @@ fn token_nl_is_the_first_token_of_a_tokenized_newline(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -109,7 +109,7 @@ fn token_nl_is_the_first_token_of_a_tokenized_newline(fixture: &LlamaFixture<'_> )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -117,7 +117,7 @@ fn token_nl_is_the_first_token_of_a_tokenized_newline(fixture: &LlamaFixture<'_> )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -125,7 +125,7 @@ fn token_nl_is_the_first_token_of_a_tokenized_newline(fixture: &LlamaFixture<'_> )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -137,7 +137,7 @@ fn decoder_only_model_has_no_decoder_start_token(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -145,7 +145,7 @@ fn decoder_only_model_has_no_decoder_start_token(fixture: &LlamaFixture<'_>) { )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -153,7 +153,7 @@ fn decoder_only_model_has_no_decoder_start_token(fixture: &LlamaFixture<'_>) { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -161,7 +161,7 @@ fn decoder_only_model_has_no_decoder_start_token(fixture: &LlamaFixture<'_>) { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -173,7 +173,7 @@ fn chat_model_has_no_separator_token(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 128, @@ -185,7 +185,7 @@ fn deepseek_r1_8b_adds_a_bos_token(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/state_snapshots.rs b/llama-cpp-bindings-tests/tests/state_snapshots.rs index bd08bd8fe..dbf8b6aab 100644 --- a/llama-cpp-bindings-tests/tests/state_snapshots.rs +++ b/llama-cpp-bindings-tests/tests/state_snapshots.rs @@ -11,7 +11,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -19,7 +19,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -27,7 +27,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -35,7 +35,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -51,7 +51,7 @@ fn a_fresh_context_reports_a_nonempty_state_size(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -59,7 +59,7 @@ fn a_fresh_context_reports_a_nonempty_state_size(fixture: &LlamaFixture<'_>) -> )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -67,7 +67,7 @@ fn a_fresh_context_reports_a_nonempty_state_size(fixture: &LlamaFixture<'_>) -> )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -75,7 +75,7 @@ fn a_fresh_context_reports_a_nonempty_state_size(fixture: &LlamaFixture<'_>) -> )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -109,7 +109,7 @@ fn state_set_data_rejects_a_truncated_snapshot(fixture: &LlamaFixture<'_>) -> Re #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -144,7 +144,7 @@ fn state_set_data_consumes_exactly_the_bytes_state_get_data_wrote( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -152,7 +152,7 @@ fn state_set_data_consumes_exactly_the_bytes_state_get_data_wrote( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -160,7 +160,7 @@ fn state_set_data_consumes_exactly_the_bytes_state_get_data_wrote( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -168,7 +168,7 @@ fn state_set_data_consumes_exactly_the_bytes_state_get_data_wrote( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/token_attributes.rs b/llama-cpp-bindings-tests/tests/token_attributes.rs index 62852b5a0..f40041fed 100644 --- a/llama-cpp-bindings-tests/tests/token_attributes.rs +++ b/llama-cpp-bindings-tests/tests/token_attributes.rs @@ -7,7 +7,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -15,7 +15,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -23,7 +23,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -31,7 +31,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/token_candidates.rs b/llama-cpp-bindings-tests/tests/token_candidates.rs index c6da5f933..a3a3fcfbd 100644 --- a/llama-cpp-bindings-tests/tests/token_candidates.rs +++ b/llama-cpp-bindings-tests/tests/token_candidates.rs @@ -13,7 +13,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 2048, @@ -47,7 +47,7 @@ fn a_token_data_array_holds_one_candidate_per_vocabulary_token( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 256, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/tokenization.rs b/llama-cpp-bindings-tests/tests/tokenization.rs index 8188de851..12f57246b 100644 --- a/llama-cpp-bindings-tests/tests/tokenization.rs +++ b/llama-cpp-bindings-tests/tests/tokenization.rs @@ -9,7 +9,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -17,7 +17,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -25,7 +25,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -33,7 +33,7 @@ use llama_cpp_test_harness_macros::llama_test; )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -56,7 +56,7 @@ fn the_first_token_of_tokenized_text_decodes_to_its_first_word( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -83,7 +83,7 @@ fn str_to_token_prepends_bos_for_a_model_that_adds_it(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -105,7 +105,7 @@ fn str_to_token_adds_nothing_for_a_model_that_adds_no_bos( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -113,7 +113,7 @@ fn str_to_token_adds_nothing_for_a_model_that_adds_no_bos( )] #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -121,7 +121,7 @@ fn str_to_token_adds_nothing_for_a_model_that_adds_no_bos( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -129,7 +129,7 @@ fn str_to_token_adds_nothing_for_a_model_that_adds_no_bos( )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -155,7 +155,7 @@ fn str_to_token_regrows_its_buffer_for_more_tokens_than_it_estimated( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/tool_call_classification.rs b/llama-cpp-bindings-tests/tests/tool_call_classification.rs index b8226e422..b9965ff11 100644 --- a/llama-cpp-bindings-tests/tests/tool_call_classification.rs +++ b/llama-cpp-bindings-tests/tests/tool_call_classification.rs @@ -30,7 +30,7 @@ fn raw_text_where(outcomes: &[IngestOutcome], belongs: impl Fn(&SampledToken) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/tool_call_grammar.rs b/llama-cpp-bindings-tests/tests/tool_call_grammar.rs index 375f20594..62b31e3d0 100644 --- a/llama-cpp-bindings-tests/tests/tool_call_grammar.rs +++ b/llama-cpp-bindings-tests/tests/tool_call_grammar.rs @@ -36,7 +36,7 @@ fn generated_tool_call_answer( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, @@ -52,7 +52,7 @@ fn a_chat_session_without_tools_needs_no_grammar_sampler(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, @@ -68,7 +68,7 @@ fn a_tool_choice_of_none_needs_no_grammar_sampler(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, @@ -91,7 +91,7 @@ fn qwen36_required_tool_call_grammar_constrains_the_answer_to_a_tool_call( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, diff --git a/llama-cpp-bindings-tests/tests/tool_call_parsing.rs b/llama-cpp-bindings-tests/tests/tool_call_parsing.rs index b11e72b59..cac715f2c 100644 --- a/llama-cpp-bindings-tests/tests/tool_call_parsing.rs +++ b/llama-cpp-bindings-tests/tests/tool_call_parsing.rs @@ -76,7 +76,7 @@ fn assert_parses_to_paris_weather_call(fixture: &LlamaFixture<'_>, answer: &str) #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -91,7 +91,7 @@ fn gemma4_parses_its_tool_call_format(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -106,7 +106,7 @@ fn glm47_parses_its_tool_call_format(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -121,7 +121,7 @@ fn mistral3_parses_its_tool_call_format(fixture: &LlamaFixture<'_>) -> Result<() #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -133,7 +133,7 @@ fn qwen35_parses_its_tool_call_format(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -184,7 +184,7 @@ tuna\n\ #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -219,7 +219,7 @@ fn qwen35_parses_a_tool_call_cut_inside_its_arguments_as_a_named_call( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -267,7 +267,7 @@ fn qwen35_parses_consecutive_tool_calls_in_order(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -275,7 +275,7 @@ fn qwen35_parses_consecutive_tool_calls_in_order(fixture: &LlamaFixture<'_>) -> )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -303,7 +303,7 @@ fn plain_content_answering_a_tool_request_parses_without_tool_calls( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, diff --git a/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs b/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs index 6466e6506..98fa1bbe5 100644 --- a/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs +++ b/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs @@ -30,7 +30,7 @@ fn most_likely_token(context: &LlamaContext<'_>, token_index: i32) -> Result, token_index: i32) -> Result, token_index: i32) -> Result, token_index: i32) -> Result syn::Result { } fn require_negated_i32_lit(literal: &Lit, field: &str) -> syn::Result { - if let Lit::Int(int_literal) = literal { - format!("-{}", int_literal.base10_digits()) - .parse::() - .map_err(|parse_error| { - syn::Error::new_spanned( - literal, - format!("field `{field}` expects an integer that fits in i32: {parse_error}"), - ) - }) - } else { - Err(syn::Error::new_spanned( + let Lit::Int(int_literal) = literal else { + return Err(syn::Error::new_spanned( literal, format!("field `{field}` expects an integer literal"), - )) - } + )); + }; + let magnitude = int_literal.base10_parse::().map_err(|parse_error| { + syn::Error::new_spanned( + literal, + format!("field `{field}` expects an integer that fits in i32: {parse_error}"), + ) + })?; + + 0_i32.checked_sub_unsigned(magnitude).ok_or_else(|| { + syn::Error::new_spanned( + literal, + format!("field `{field}` expects an integer that fits in i32: -{magnitude} is below its minimum"), + ) + }) } fn require_i32_value(expression: &Expr, field: &str) -> syn::Result { @@ -494,7 +498,25 @@ mod tests { parse(source) .expect_err("an i32 below i32::MIN must fail") .to_string(), - "field `n_gpu_layers` expects an integer that fits in i32: number too small to fit in target type" + "field `n_gpu_layers` expects an integer that fits in i32: -2147483649 is below its minimum" + ); + } + + #[test] + fn a_negative_i32_field_whose_magnitude_exceeds_u32_is_rejected() { + let source = "\ + model_source = HuggingFace(\"x\", \"y\"), \ + n_gpu_layers = -4294967296, \ + load_mode = Mmap, \ + n_ctx = 1, \ + n_batch = 1, \ + n_ubatch = 1"; + + assert_eq!( + parse(source) + .expect_err("a magnitude beyond u32 must fail") + .to_string(), + "field `n_gpu_layers` expects an integer that fits in i32: number too large to fit in target type" ); } diff --git a/llama-cpp-test-harness/tests/phase_conclusions.rs b/llama-cpp-test-harness/tests/phase_conclusions.rs index e1fadfed6..2dea0f8a8 100644 --- a/llama-cpp-test-harness/tests/phase_conclusions.rs +++ b/llama-cpp-test-harness/tests/phase_conclusions.rs @@ -11,7 +11,7 @@ use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -23,7 +23,7 @@ const fn phase_a_passing_trial_that_only_needs_its_model_to_load(fixture: &Llama #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -35,7 +35,7 @@ fn phase_a_second_passing_trial(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -47,7 +47,7 @@ fn phase_a_intentionally_failing_trial(_fixture: &LlamaFixture<'_>) -> Result<() #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -59,7 +59,7 @@ fn phase_b_first_passing_trial(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -71,7 +71,7 @@ fn phase_b_second_passing_trial(fixture: &LlamaFixture<'_>) { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -87,7 +87,7 @@ fn phase_b_second_passing_trial(fixture: &LlamaFixture<'_>) { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -96,7 +96,7 @@ fn phase_b_second_passing_trial(fixture: &LlamaFixture<'_>) { )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -117,7 +117,7 @@ fn phase_a_trial_also_scheduled_into_every_setup_failure_phase(fixture: &LlamaFi #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, From 4af3315f995a19857d81456187ab6502f3474362 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 08:51:45 +0100 Subject: [PATCH 73/82] Include ggml-backend.h where the device wrappers use it --- llama-cpp-bindings-sys/wrapper_common.cpp | 1 + llama-cpp-bindings-sys/wrapper_common.h | 1 + 2 files changed, 2 insertions(+) diff --git a/llama-cpp-bindings-sys/wrapper_common.cpp b/llama-cpp-bindings-sys/wrapper_common.cpp index fc0c1f886..57e23a535 100644 --- a/llama-cpp-bindings-sys/wrapper_common.cpp +++ b/llama-cpp-bindings-sys/wrapper_common.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include "wrapper_report_current_exception.h" #include "wrapper_utils.h" diff --git a/llama-cpp-bindings-sys/wrapper_common.h b/llama-cpp-bindings-sys/wrapper_common.h index 64c1972e3..857a3d313 100644 --- a/llama-cpp-bindings-sys/wrapper_common.h +++ b/llama-cpp-bindings-sys/wrapper_common.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include From ebbce47b3cd7afe8c2ae5856dc3c2be36535edc4 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 09:01:38 +0100 Subject: [PATCH 74/82] Test template availability on the model and memory absence on valid sequence ids --- .../tests/chat_session_llama_cpp_failures.rs | 11 +++-------- llama-cpp-bindings-tests/tests/memory_availability.rs | 3 ++- llama-cpp-bindings/src/error/chat_session_error.rs | 3 --- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs b/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs index 574501cbf..95125263c 100644 --- a/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs +++ b/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs @@ -8,7 +8,6 @@ use llama_cpp_bindings::error::chat_templates_error::ChatTemplatesError; use llama_cpp_bindings::error::grammar_error::GrammarError; use llama_cpp_bindings::error::parse_chat_message_error::ParseChatMessageError; use llama_cpp_bindings::error::sampler_accept_error::SamplerAcceptError; -use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; use llama_cpp_bindings_tests::weather_chat_session::weather_chat_session; use llama_cpp_fault_injection::armed_fault::ArmedFault; use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; @@ -30,14 +29,10 @@ fn not_utf8() -> Utf8Error { n_batch = 512, n_ubatch = 512, )] -fn a_chat_session_needs_a_model_with_a_chat_template(fixture: &LlamaFixture<'_>) { +fn a_model_without_a_chat_template_has_no_chat_templates(fixture: &LlamaFixture<'_>) { assert_eq!( - user_turn_chat_session(fixture.model, "hello", "[]", false) - .err() - .map(|failure| failure.downcast::().ok()), - Some(Some(ChatSessionError::Templates( - ChatTemplatesError::ModelHasNoChatTemplate - ))) + fixture.model.chat_templates().err(), + Some(ChatTemplatesError::ModelHasNoChatTemplate) ); } diff --git a/llama-cpp-bindings-tests/tests/memory_availability.rs b/llama-cpp-bindings-tests/tests/memory_availability.rs index 0c55d49c2..77a06015d 100644 --- a/llama-cpp-bindings-tests/tests/memory_availability.rs +++ b/llama-cpp-bindings-tests/tests/memory_availability.rs @@ -8,6 +8,7 @@ use llama_cpp_bindings::error::memory_seq_cp_error::MemorySeqCpError; use llama_cpp_bindings::error::memory_seq_div_error::MemorySeqDivError; use llama_cpp_bindings::error::memory_seq_pos_max_error::MemorySeqPosMaxError; use llama_cpp_bindings::error::memory_seq_rm_error::MemorySeqRmError; +use llama_cpp_bindings_tests::two_sequence_context::two_sequence_context; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -77,7 +78,7 @@ fn memory_edits_without_a_memory_module_trivially_succeed( fn memory_operations_needing_a_memory_module_report_its_absence( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let mut context = fixture.build_context()?; + let mut context = two_sequence_context(fixture)?; let divisor = NonZeroU8::new(2).context("2 is non-zero")?; assert_eq!( diff --git a/llama-cpp-bindings/src/error/chat_session_error.rs b/llama-cpp-bindings/src/error/chat_session_error.rs index e3689448c..593c01b1b 100644 --- a/llama-cpp-bindings/src/error/chat_session_error.rs +++ b/llama-cpp-bindings/src/error/chat_session_error.rs @@ -5,7 +5,6 @@ use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; -use crate::error::chat_templates_error::ChatTemplatesError; use crate::error::grammar_error::GrammarError; use crate::error::sampler_accept_error::SamplerAcceptError; use crate::error::string_to_token_error::StringToTokenError; @@ -18,8 +17,6 @@ pub enum ChatSessionError { FfiStatus(#[from] FfiStatusError), #[error(transparent)] FfiContract(#[from] FfiContractError), - #[error("the model's chat templates are unavailable: {0}")] - Templates(#[from] ChatTemplatesError), #[error("a chat message is not valid UTF-8")] MessageNotUtf8(#[source] Utf8Error), #[error("the chat request contains an interior NUL byte")] From 5b09ee411087a75e81da391270940f77989a9865 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 10:22:17 +0100 Subject: [PATCH 75/82] Cover a refused copy source, removal from every sequence and backend loading from a directory, and write pieces as an outcome --- .../src/user_turn_chat_session.rs | 8 +++- .../tests/memory_sequence_edits.rs | 21 ++++++++++ .../tests/memory_sequence_id_bounds.rs | 26 ++++++++++++ .../src/load_backends_from_path.rs | 13 ++++++ llama-cpp-bindings/src/model.rs | 40 ++++++++----------- llama-cpp-bindings/src/model/piece_write.rs | 36 +++++++++++++++++ 6 files changed, 120 insertions(+), 24 deletions(-) create mode 100644 llama-cpp-bindings/src/model/piece_write.rs diff --git a/llama-cpp-bindings-tests/src/user_turn_chat_session.rs b/llama-cpp-bindings-tests/src/user_turn_chat_session.rs index 72805e2c4..47fd7efb7 100644 --- a/llama-cpp-bindings-tests/src/user_turn_chat_session.rs +++ b/llama-cpp-bindings-tests/src/user_turn_chat_session.rs @@ -12,6 +12,10 @@ use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; /// # Errors /// /// Forwards message construction and chat session errors. +/// +/// # Panics +/// +/// Panics when the model has no chat templates. pub fn user_turn_chat_session<'model>( model: &'model LlamaModel, content: &str, @@ -20,7 +24,9 @@ pub fn user_turn_chat_session<'model>( ) -> Result> { Ok(ChatSession::new( model, - model.chat_templates()?, + model + .chat_templates() + .expect("the model must have chat templates"), &[LlamaChatMessage::new( "user".to_owned(), content.to_owned(), diff --git a/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs b/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs index 7adb081ea..552c557b8 100644 --- a/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs +++ b/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs @@ -162,3 +162,24 @@ fn qwen35_recurrent_memory_refuses_to_remove_part_of_a_sequence( Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn removing_from_every_sequence_empties_the_primed_kv_cache( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_memory(fixture, &mut context)?; + + assert_eq!(context.memory_seq_rm(None, None, None), Ok(())); + assert_eq!(context.memory_seq_pos_max(0)?, -1); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs b/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs index dd6204bd1..9a2a10784 100644 --- a/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs +++ b/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs @@ -233,3 +233,29 @@ fn memory_seq_rm_rejects_a_sequence_outside_the_context_sequences( Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_cp_rejects_a_source_outside_the_context_sequences( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + assert_eq!( + context.memory_seq_cp(1, 0, None, None), + Err(MemorySeqCpError::SequenceIdOutOfRange( + SequenceIdOutOfRange { + seq_id: 1, + n_seq_max: 1 + } + )) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings/src/load_backends_from_path.rs b/llama-cpp-bindings/src/load_backends_from_path.rs index fbb287667..2658b627b 100644 --- a/llama-cpp-bindings/src/load_backends_from_path.rs +++ b/llama-cpp-bindings/src/load_backends_from_path.rs @@ -27,6 +27,8 @@ mod tests { use std::os::unix::ffi::OsStrExt; use std::path::PathBuf; + use std::env; + use super::load_backends_from_path; use crate::load_backends_error::LoadBackendsError; @@ -51,4 +53,15 @@ mod tests { Err(LoadBackendsError::PathNotUtf8(path.clone())) ); } + + #[test] + fn a_directory_without_backend_libraries_loads_no_backend() { + let registered_before = unsafe { llama_cpp_bindings_sys::ggml_backend_reg_count() }; + + assert_eq!(load_backends_from_path(&env::temp_dir()), Ok(())); + assert_eq!( + unsafe { llama_cpp_bindings_sys::ggml_backend_reg_count() }, + registered_before + ); + } } diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index 8ab2a464b..e2309019f 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -10,6 +10,7 @@ pub mod llama_split_mode_parse_error; pub mod lora_adapter_scale; pub mod params; pub mod parse_special_tokens; +mod piece_write; pub mod rope_type; pub mod split_mode; pub mod tokenizer_input; @@ -59,6 +60,7 @@ use crate::model::add_bos::AddBos; use crate::model::llama_chat_template::LlamaChatTemplate; use crate::model::llama_lora_adapter::LlamaLoraAdapter; use crate::model::parse_special_tokens::ParseSpecialTokens; +use crate::model::piece_write::PieceWrite; use crate::model::rope_type::RopeType; use crate::model::tokenizer_input::TokenizerInput; use crate::model::vocab_type::VocabType; @@ -381,11 +383,16 @@ impl LlamaModel { let token = self.require_vocabulary_token(token)?; let buffer_len = c_int::try_from(buffer_size)?; let mut buffer: Vec = vec![0u8; buffer_size]; - let written = self.write_piece(token, &mut buffer, buffer_len, special, lstrip)?; + match self.write_piece(token, &mut buffer, buffer_len, special, lstrip) { + PieceWrite::Written(written) => { + buffer.truncate(written); - buffer.truncate(written); - - Ok(buffer) + Ok(buffer) + } + PieceWrite::NeedsBuffer(negated_size) => { + Err(TokenToStringError::InsufficientBufferSpace(negated_size)) + } + } } fn write_piece( @@ -395,8 +402,8 @@ impl LlamaModel { buffer_len: c_int, special: bool, lstrip: Option, - ) -> Result { - let size = unsafe { + ) -> PieceWrite { + PieceWrite::from(unsafe { llama_cpp_bindings_sys::llama_rs_token_to_piece( self.vocab, token.0, @@ -405,14 +412,7 @@ impl LlamaModel { lstrip.map_or(0, |strip_count| i32::from(strip_count.get())), special, ) - }; - - match size { - error_code if error_code.is_negative() => { - Err(TokenToStringError::InsufficientBufferSpace(error_code)) - } - size => Ok(size.unsigned_abs() as usize), - } + }) } fn read_piece( @@ -426,16 +426,10 @@ impl LlamaModel { let mut probe = [0_u8; PIECE_PROBE_SIZE]; match self.write_piece(token, &mut probe, PIECE_PROBE_LEN, special, lstrip) { - Ok(written) => Ok(read(probe.split_at(written).0)), - Err(TokenToStringError::InsufficientBufferSpace(required_size)) => self - .token_to_piece_bytes( - token, - required_size.unsigned_abs() as usize, - special, - lstrip, - ) + PieceWrite::Written(written) => Ok(read(probe.split_at(written).0)), + PieceWrite::NeedsBuffer(negated_size) => self + .token_to_piece_bytes(token, negated_size.unsigned_abs() as usize, special, lstrip) .map(|piece| read(&piece)), - Err(probe_error) => Err(probe_error), } } diff --git a/llama-cpp-bindings/src/model/piece_write.rs b/llama-cpp-bindings/src/model/piece_write.rs new file mode 100644 index 000000000..3aea97eee --- /dev/null +++ b/llama-cpp-bindings/src/model/piece_write.rs @@ -0,0 +1,36 @@ +use std::ffi::c_int; + +/// What `llama_token_to_piece` did with the buffer it was given: wrote the piece's bytes, or +/// reported, as a negative size, the bytes the piece needs. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PieceWrite { + Written(usize), + NeedsBuffer(c_int), +} + +impl From for PieceWrite { + fn from(size: c_int) -> Self { + if size.is_negative() { + Self::NeedsBuffer(size) + } else { + Self::Written(size.unsigned_abs() as usize) + } + } +} + +#[cfg(test)] +mod tests { + use super::PieceWrite; + + #[test] + fn a_negative_size_asks_for_a_larger_buffer_and_any_other_is_the_written_length() { + assert_eq!( + [-7, 0, 5].map(PieceWrite::from), + [ + PieceWrite::NeedsBuffer(-7), + PieceWrite::Written(0), + PieceWrite::Written(5) + ] + ); + } +} From 124775c760e723ba1fbac13964a97a8889b753e4 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 15:39:05 +0100 Subject: [PATCH 76/82] Link the OpenMP runtime CMake resolved for ggml by its exact file name instead of guessing gomp --- Cargo.lock | 2 + llama-cpp-bindings-build/Cargo.toml | 2 + .../.cmake/api/v1/reply/index-0.json | 1 + .../.cmake/api/v1/reply/index-0.json | 1 + .../.cmake/api/v1/reply/cache-v2-0.json | 1 + .../fixtures/cmake_project/CMakeLists.txt | 7 + llama-cpp-bindings-build/src/build_context.rs | 61 ++--- llama-cpp-bindings-build/src/build_error.rs | 37 +++ .../src/cargo_directive.rs | 12 +- llama-cpp-bindings-build/src/cmake_build.rs | 14 +- llama-cpp-bindings-build/src/cmake_cache.rs | 48 ++++ .../src/cmake_cache_entry.rs | 8 + .../src/cmake_file_api.rs | 150 +++++++++++ llama-cpp-bindings-build/src/cmake_replies.rs | 10 + .../src/cmake_reply_file.rs | 10 + .../src/cmake_reply_index.rs | 9 + llama-cpp-bindings-build/src/lib.rs | 8 + .../src/link_directives.rs | 30 +-- llama-cpp-bindings-build/src/native_build.rs | 77 +++++- .../src/openmp_linking.rs | 120 +++++++++ .../src/openmp_runtime_source.rs | 235 ++++++++++++++++++ 21 files changed, 768 insertions(+), 75 deletions(-) create mode 100644 llama-cpp-bindings-build/fixtures/cmake_file_api/index_naming_a_missing_cache/.cmake/api/v1/reply/index-0.json create mode 100644 llama-cpp-bindings-build/fixtures/cmake_file_api/index_without_cache_reply/.cmake/api/v1/reply/index-0.json create mode 100644 llama-cpp-bindings-build/fixtures/cmake_file_api/reply_without_index/.cmake/api/v1/reply/cache-v2-0.json create mode 100644 llama-cpp-bindings-build/src/cmake_cache.rs create mode 100644 llama-cpp-bindings-build/src/cmake_cache_entry.rs create mode 100644 llama-cpp-bindings-build/src/cmake_file_api.rs create mode 100644 llama-cpp-bindings-build/src/cmake_replies.rs create mode 100644 llama-cpp-bindings-build/src/cmake_reply_file.rs create mode 100644 llama-cpp-bindings-build/src/cmake_reply_index.rs create mode 100644 llama-cpp-bindings-build/src/openmp_linking.rs create mode 100644 llama-cpp-bindings-build/src/openmp_runtime_source.rs diff --git a/Cargo.lock b/Cargo.lock index b1a70dfbd..406c19b58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1195,6 +1195,8 @@ dependencies = [ "enumflags2", "find_cuda_helper", "llama-cpp-wrapper-sources", + "serde", + "serde_json", "thiserror", ] diff --git a/llama-cpp-bindings-build/Cargo.toml b/llama-cpp-bindings-build/Cargo.toml index 4bfc4d5a1..3aae18bbd 100644 --- a/llama-cpp-bindings-build/Cargo.toml +++ b/llama-cpp-bindings-build/Cargo.toml @@ -13,6 +13,8 @@ cmake = { workspace = true } enumflags2 = { workspace = true } find_cuda_helper = { workspace = true } llama-cpp-wrapper-sources = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } thiserror = { workspace = true } [features] diff --git a/llama-cpp-bindings-build/fixtures/cmake_file_api/index_naming_a_missing_cache/.cmake/api/v1/reply/index-0.json b/llama-cpp-bindings-build/fixtures/cmake_file_api/index_naming_a_missing_cache/.cmake/api/v1/reply/index-0.json new file mode 100644 index 000000000..1d183cfb6 --- /dev/null +++ b/llama-cpp-bindings-build/fixtures/cmake_file_api/index_naming_a_missing_cache/.cmake/api/v1/reply/index-0.json @@ -0,0 +1 @@ +{"reply":{"cache-v2":{"kind":"cache","version":{"major":2,"minor":0},"jsonFile":"cache-v2-missing.json"}}} diff --git a/llama-cpp-bindings-build/fixtures/cmake_file_api/index_without_cache_reply/.cmake/api/v1/reply/index-0.json b/llama-cpp-bindings-build/fixtures/cmake_file_api/index_without_cache_reply/.cmake/api/v1/reply/index-0.json new file mode 100644 index 000000000..ee5d64f1f --- /dev/null +++ b/llama-cpp-bindings-build/fixtures/cmake_file_api/index_without_cache_reply/.cmake/api/v1/reply/index-0.json @@ -0,0 +1 @@ +{"reply":{}} diff --git a/llama-cpp-bindings-build/fixtures/cmake_file_api/reply_without_index/.cmake/api/v1/reply/cache-v2-0.json b/llama-cpp-bindings-build/fixtures/cmake_file_api/reply_without_index/.cmake/api/v1/reply/cache-v2-0.json new file mode 100644 index 000000000..f5cae2c6b --- /dev/null +++ b/llama-cpp-bindings-build/fixtures/cmake_file_api/reply_without_index/.cmake/api/v1/reply/cache-v2-0.json @@ -0,0 +1 @@ +{"kind":"cache","version":{"major":2,"minor":0},"entries":[]} diff --git a/llama-cpp-bindings-build/fixtures/cmake_project/CMakeLists.txt b/llama-cpp-bindings-build/fixtures/cmake_project/CMakeLists.txt index 3ad4aca44..b58105960 100644 --- a/llama-cpp-bindings-build/fixtures/cmake_project/CMakeLists.txt +++ b/llama-cpp-bindings-build/fixtures/cmake_project/CMakeLists.txt @@ -8,3 +8,10 @@ endif() add_library(${FIXTURE_LIBRARY_NAME} STATIC fixture.c) install(TARGETS ${FIXTURE_LIBRARY_NAME} DESTINATION lib) + +set(GGML_OPENMP_ENABLED "ON" CACHE INTERNAL "") +set(OpenMP_C_LIB_NAMES "${FIXTURE_LIBRARY_NAME}" CACHE STRING "") +set(OpenMP_CXX_LIB_NAMES "${FIXTURE_LIBRARY_NAME}" CACHE STRING "") +set(OpenMP_${FIXTURE_LIBRARY_NAME}_LIBRARY + "${CMAKE_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${FIXTURE_LIBRARY_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" + CACHE FILEPATH "") diff --git a/llama-cpp-bindings-build/src/build_context.rs b/llama-cpp-bindings-build/src/build_context.rs index 496ea90b3..91b982e0d 100644 --- a/llama-cpp-bindings-build/src/build_context.rs +++ b/llama-cpp-bindings-build/src/build_context.rs @@ -16,7 +16,6 @@ pub struct BuildContext { pub target_triple: String, pub host_triple: String, pub cargo_cfg_target_arch: String, - pub cargo_cfg_target_env: String, pub build_shared_libs: bool, pub opt_level: String, pub debug_info: String, @@ -46,9 +45,6 @@ impl BuildContext { host_os, )?, host_triple: environment.required("HOST")?, - cargo_cfg_target_env: environment - .optional("CARGO_CFG_TARGET_ENV")? - .unwrap_or_default(), build_shared_libs: environment.has(BuildFeature::DynamicLink), opt_level: environment.required("OPT_LEVEL")?, debug_info: environment.required("DEBUG")?, @@ -63,6 +59,11 @@ impl BuildContext { self.out_dir.join("cmake") } + #[must_use] + pub fn cmake_build_dir(&self) -> PathBuf { + self.cmake_dir().join("build") + } + #[must_use] pub fn llama_src(&self) -> PathBuf { self.sys_dir.join("llama.cpp") @@ -100,9 +101,8 @@ pub mod tests { use crate::host_os::HostOs; use crate::target_os::TargetOs; - const CARGO_VARIABLES: [(&str, &str); 9] = [ + const CARGO_VARIABLES: [(&str, &str); 8] = [ ("CARGO_CFG_TARGET_ARCH", "x86_64"), - ("CARGO_CFG_TARGET_ENV", "gnu"), ("CARGO_CFG_TARGET_OS", "linux"), ("CARGO_MANIFEST_DIR", "/sys"), ("DEBUG", "false"), @@ -156,7 +156,6 @@ pub mod tests { target_triple: "x86_64-unknown-linux-gnu".to_owned(), host_triple: "x86_64-unknown-linux-gnu".to_owned(), cargo_cfg_target_arch: "x86_64".to_owned(), - cargo_cfg_target_env: "gnu".to_owned(), build_shared_libs: true, opt_level: "3".to_owned(), debug_info: "false".to_owned(), @@ -166,7 +165,7 @@ pub mod tests { } #[test] - fn every_variable_cargo_sets_for_build_scripts_except_the_target_env_is_required() { + fn every_variable_cargo_sets_for_build_scripts_is_required() { let without = |name: &str| { let mut environment = cargo_environment(BitFlags::empty(), &[]); @@ -176,42 +175,30 @@ pub mod tests { .err() .map(|error| error.to_string()) }; - let required = CARGO_VARIABLES - .map(|(name, _)| name) - .into_iter() - .filter(|name| *name != "CARGO_CFG_TARGET_ENV") - .collect::>(); + let required = CARGO_VARIABLES.map(|(name, _)| name); assert_eq!( - required - .iter() - .map(|name| without(name)) - .collect::>>(), - required - .iter() - .map(|name| Some(format!( - "environment variable {name} could not be read: environment variable not \ - found" - ))) - .collect::>>() + required.map(without), + required.map(|name| Some(format!( + "environment variable {name} could not be read: environment variable not found" + ))) ); } #[test] - fn optional_variables_that_are_not_unicode_are_refused() { + fn a_build_debug_request_that_is_not_unicode_is_refused() { assert_eq!( - ["CARGO_CFG_TARGET_ENV", "BUILD_DEBUG"].map(|name| { - BuildContext::from_environment( - &with_non_unicode(cargo_environment(BitFlags::empty(), &[]), name), - HostOs::Linux, - ) - .err() - .map(|error| error.to_string()) - }), - ["CARGO_CFG_TARGET_ENV", "BUILD_DEBUG"].map(|name| Some(format!( - "environment variable {name} could not be read: environment variable was not \ - valid unicode: \"\\xFF\"" - ))) + BuildContext::from_environment( + &with_non_unicode(cargo_environment(BitFlags::empty(), &[]), "BUILD_DEBUG"), + HostOs::Linux, + ) + .err() + .map(|error| error.to_string()), + Some( + "environment variable BUILD_DEBUG could not be read: environment variable was \ + not valid unicode: \"\\xFF\"" + .to_owned() + ) ); } diff --git a/llama-cpp-bindings-build/src/build_error.rs b/llama-cpp-bindings-build/src/build_error.rs index 0c10d323e..9aa6d3c97 100644 --- a/llama-cpp-bindings-build/src/build_error.rs +++ b/llama-cpp-bindings-build/src/build_error.rs @@ -73,4 +73,41 @@ pub enum BuildError { EnvironmentDirectory { name: &'static str, path: PathBuf }, #[error("cargo directives could not be written: {0}")] DirectiveWrite(#[source] io::Error), + #[error("the CMake file API query could not be written into {path}: {source}")] + CmakeFileApiQueryNotWritten { + path: PathBuf, + #[source] + source: io::Error, + }, + #[error("the CMake file API reply directory {path} could not be listed: {source}")] + CmakeFileApiReplyDirectoryUnreadable { + path: PathBuf, + #[source] + source: io::Error, + }, + #[error("CMake wrote no file API reply index into {path}")] + CmakeFileApiReplyIndexMissing { path: PathBuf }, + #[error("the CMake file API reply file {path} could not be read: {source}")] + CmakeFileApiReplyFileUnreadable { + path: PathBuf, + #[source] + source: io::Error, + }, + #[error("the CMake file API reply file {path} is not the reply this crate queried: {source}")] + CmakeFileApiReplyFileInvalid { + path: PathBuf, + #[source] + source: serde_json::Error, + }, + #[error("the CMake cache has no entry {name}")] + CmakeCacheEntryMissing { name: String }, + #[error("the CMake cache entry {name} is {value}, neither ON nor OFF")] + CmakeCacheEntryNotASwitch { name: &'static str, value: String }, + #[error( + "ggml needs the OpenMP runtime {libraries:?}, which Apple's linker cannot link by its \ + exact file name" + )] + OpenmpRuntimeUnlinkableOnApple { libraries: Vec }, + #[error("the OpenMP runtime library {library} names no file")] + OpenmpRuntimeLibraryWithoutFileName { library: PathBuf }, } diff --git a/llama-cpp-bindings-build/src/cargo_directive.rs b/llama-cpp-bindings-build/src/cargo_directive.rs index 901d21e31..680a19f81 100644 --- a/llama-cpp-bindings-build/src/cargo_directive.rs +++ b/llama-cpp-bindings-build/src/cargo_directive.rs @@ -12,6 +12,7 @@ pub enum CargoDirective { LinkSearch(PathBuf), LinkSearchNative(PathBuf), LinkLibrary(NativeLibrary), + LinkDylibByFileName(PathBuf), Debug(String), } @@ -40,6 +41,11 @@ impl Display for CargoDirective { ) } Self::LinkLibrary(library) => write!(formatter, "cargo:rustc-link-lib={library}"), + Self::LinkDylibByFileName(file_name) => write!( + formatter, + "cargo:rustc-link-lib=dylib:+verbatim={}", + file_name.display() + ), Self::Debug(message) => write!(formatter, "cargo:warning=[DEBUG] {message}"), } } @@ -53,7 +59,7 @@ mod tests { use crate::link_kind::LinkKind; use crate::native_library::NativeLibrary; - fn every_directive() -> [CargoDirective; 6] { + fn every_directive() -> [CargoDirective; 7] { [ CargoDirective::RerunIfChanged(PathBuf::from("wrapper.h")), CargoDirective::RerunIfEnvChanged("CUDAARCHS".to_owned()), @@ -63,6 +69,7 @@ mod tests { link_kind: LinkKind::Static, name: "llama", }), + CargoDirective::LinkDylibByFileName(PathBuf::from("libgomp.so")), CargoDirective::Debug("TARGET: x86_64-unknown-linux-gnu".to_owned()), ] } @@ -77,6 +84,7 @@ mod tests { "cargo:rustc-link-search=/out/lib", "cargo:rustc-link-search=native=/out/build/common", "cargo:rustc-link-lib=static=llama", + "cargo:rustc-link-lib=dylib:+verbatim=libgomp.so", "cargo:warning=[DEBUG] TARGET: x86_64-unknown-linux-gnu", ] ); @@ -86,7 +94,7 @@ mod tests { fn only_debug_messages_are_debug_directives() { assert_eq!( every_directive().map(|directive| directive.is_debug()), - [false, false, false, false, false, true] + [false, false, false, false, false, false, true] ); } } diff --git a/llama-cpp-bindings-build/src/cmake_build.rs b/llama-cpp-bindings-build/src/cmake_build.rs index e81efed61..5e914e213 100644 --- a/llama-cpp-bindings-build/src/cmake_build.rs +++ b/llama-cpp-bindings-build/src/cmake_build.rs @@ -4,14 +4,18 @@ use std::path::PathBuf; use cmake::Config; use crate::build_error::BuildError; +use crate::cmake_cache::CmakeCache; use crate::cmake_define::CmakeDefine; +use crate::cmake_file_api::CmakeFileApi; use crate::native_profile::NATIVE_PROFILE; -/// Configures the `CMake` project in `source_dir` and builds and installs it into `out_dir`. +/// Configures the `CMake` project in `source_dir` and builds and installs it into `out_dir`, +/// reading back through `file_api` the cache `CMake` configured it with. #[derive(Clone, Debug, Eq, PartialEq)] pub struct CmakeBuild { pub source_dir: PathBuf, pub out_dir: PathBuf, + pub file_api: CmakeFileApi, pub target_triple: String, pub host_triple: String, pub defines: Vec, @@ -22,18 +26,20 @@ impl CmakeBuild { /// # Errors /// /// Returns [`BuildError::Filesystem`] when a directory the project installs into cannot be - /// created. + /// created, or the [`BuildError`] of [`CmakeFileApi::request_cache`] or + /// [`CmakeFileApi::cache`]. /// /// # Panics /// /// Panics when `CMake` fails, the way the `cmake` crate reports a failed build. - pub fn build(&self) -> Result<(), BuildError> { + pub fn build(&self) -> Result { self.required_directories.iter().try_for_each(|directory| { fs::create_dir_all(directory).map_err(|source| BuildError::Filesystem { path: directory.clone(), source, }) })?; + self.file_api.request_cache()?; let mut config = Config::new(&self.source_dir); @@ -48,6 +54,6 @@ impl CmakeBuild { .profile(NATIVE_PROFILE) .build(); - Ok(()) + self.file_api.cache() } } diff --git a/llama-cpp-bindings-build/src/cmake_cache.rs b/llama-cpp-bindings-build/src/cmake_cache.rs new file mode 100644 index 000000000..f3c91306b --- /dev/null +++ b/llama-cpp-bindings-build/src/cmake_cache.rs @@ -0,0 +1,48 @@ +use serde::Deserialize; + +use crate::build_error::BuildError; +use crate::cmake_cache_entry::CmakeCacheEntry; + +/// The `CMake` cache of a configured build tree, read through the file API's `cache` object. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +pub struct CmakeCache { + pub entries: Vec, +} + +impl CmakeCache { + #[must_use] + pub fn contains(&self, name: &str) -> bool { + self.entries.iter().any(|entry| entry.name == name) + } + + /// # Errors + /// + /// Returns [`BuildError::CmakeCacheEntryMissing`] when the cache has no entry `name`. + pub fn value(&self, name: &str) -> Result<&str, BuildError> { + self.entries + .iter() + .find(|entry| entry.name == name) + .map(|entry| entry.value.as_str()) + .ok_or_else(|| BuildError::CmakeCacheEntryMissing { + name: name.to_owned(), + }) + } +} + +#[cfg(test)] +pub mod tests { + use super::CmakeCache; + use crate::cmake_cache_entry::CmakeCacheEntry; + + pub fn cache_of(entries: &[(&str, &str)]) -> CmakeCache { + CmakeCache { + entries: entries + .iter() + .map(|(name, value)| CmakeCacheEntry { + name: (*name).to_owned(), + value: (*value).to_owned(), + }) + .collect(), + } + } +} diff --git a/llama-cpp-bindings-build/src/cmake_cache_entry.rs b/llama-cpp-bindings-build/src/cmake_cache_entry.rs new file mode 100644 index 000000000..15ecdb9ab --- /dev/null +++ b/llama-cpp-bindings-build/src/cmake_cache_entry.rs @@ -0,0 +1,8 @@ +use serde::Deserialize; + +/// One entry of the `CMake` cache, as the file API's `cache` object lists it. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +pub struct CmakeCacheEntry { + pub name: String, + pub value: String, +} diff --git a/llama-cpp-bindings-build/src/cmake_file_api.rs b/llama-cpp-bindings-build/src/cmake_file_api.rs new file mode 100644 index 000000000..b96e4e4fd --- /dev/null +++ b/llama-cpp-bindings-build/src/cmake_file_api.rs @@ -0,0 +1,150 @@ +use std::fs; +use std::fs::DirEntry; +use std::io; +use std::path::Path; +use std::path::PathBuf; + +use serde::de::DeserializeOwned; + +use crate::build_error::BuildError; +use crate::cmake_cache::CmakeCache; +use crate::cmake_reply_index::CmakeReplyIndex; + +const REPLY_INDEX_PREFIX: &[u8] = b"index-"; + +fn read_reply_file(path: PathBuf) -> Result { + let contents = + fs::read(&path).map_err(|source| BuildError::CmakeFileApiReplyFileUnreadable { + path: path.clone(), + source, + })?; + + serde_json::from_slice(&contents) + .map_err(|source| BuildError::CmakeFileApiReplyFileInvalid { path, source }) +} + +fn current_reply_index(reply_dir: &Path) -> Result { + fs::read_dir(reply_dir) + .and_then(Iterator::collect::, io::Error>>) + .map_err(|source| BuildError::CmakeFileApiReplyDirectoryUnreadable { + path: reply_dir.to_path_buf(), + source, + })? + .into_iter() + .map(|entry| entry.file_name()) + .filter(|file_name| file_name.as_encoded_bytes().starts_with(REPLY_INDEX_PREFIX)) + .max() + .map(|file_name| reply_dir.join(file_name)) + .ok_or_else(|| BuildError::CmakeFileApiReplyIndexMissing { + path: reply_dir.to_path_buf(), + }) +} + +/// The `CMake` file API of the build tree in `build_dir`, through which the build learns what +/// `CMake` resolved while configuring. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CmakeFileApi { + pub build_dir: PathBuf, +} + +impl CmakeFileApi { + fn api_dir(&self) -> PathBuf { + self.build_dir.join(".cmake").join("api").join("v1") + } + + /// Asks `CMake` to answer with its cache the next time it configures this build tree. + /// + /// # Errors + /// + /// Returns [`BuildError::CmakeFileApiQueryNotWritten`] when the query cannot be written. + pub fn request_cache(&self) -> Result<(), BuildError> { + let query_dir = self.api_dir().join("query"); + + fs::create_dir_all(&query_dir) + .and_then(|()| fs::write(query_dir.join("cache-v2"), b"")) + .map_err(|source| BuildError::CmakeFileApiQueryNotWritten { + path: query_dir, + source, + }) + } + + /// The cache `CMake` reported in its current reply to [`Self::request_cache`]. + /// + /// # Errors + /// + /// Returns [`BuildError::CmakeFileApiReplyDirectoryUnreadable`] when `CMake` wrote no + /// reply, [`BuildError::CmakeFileApiReplyIndexMissing`] when the reply has no index, or + /// [`BuildError::CmakeFileApiReplyFileUnreadable`] and + /// [`BuildError::CmakeFileApiReplyFileInvalid`] when a reply file cannot be read as the + /// reply this crate queried. + pub fn cache(&self) -> Result { + let reply_dir = self.api_dir().join("reply"); + let CmakeReplyIndex { reply } = read_reply_file(current_reply_index(&reply_dir)?)?; + + read_reply_file(reply_dir.join(reply.cache.json_file)) + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::CmakeFileApi; + + const FIXTURES: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/fixtures/cmake_file_api"); + + fn cache_failure(build_tree: &str) -> Option { + CmakeFileApi { + build_dir: PathBuf::from(FIXTURES).join(build_tree), + } + .cache() + .err() + .map(|error| error.to_string()) + } + + #[test] + fn a_build_tree_cmake_never_replied_to_is_refused() { + assert_eq!( + cache_failure("never_replied"), + Some(format!( + "the CMake file API reply directory {FIXTURES}/never_replied/.cmake/api/v1/reply \ + could not be listed: No such file or directory (os error 2)" + )) + ); + } + + #[test] + fn a_reply_without_an_index_is_refused() { + assert_eq!( + cache_failure("reply_without_index"), + Some(format!( + "CMake wrote no file API reply index into \ + {FIXTURES}/reply_without_index/.cmake/api/v1/reply" + )) + ); + } + + #[test] + fn an_index_naming_a_missing_reply_file_is_refused() { + assert_eq!( + cache_failure("index_naming_a_missing_cache"), + Some(format!( + "the CMake file API reply file \ + {FIXTURES}/index_naming_a_missing_cache/.cmake/api/v1/reply/cache-v2-missing.json \ + could not be read: No such file or directory (os error 2)" + )) + ); + } + + #[test] + fn an_index_without_a_cache_reply_is_refused() { + assert_eq!( + cache_failure("index_without_cache_reply"), + Some(format!( + "the CMake file API reply file \ + {FIXTURES}/index_without_cache_reply/.cmake/api/v1/reply/index-0.json is not the \ + reply this crate queried: missing field `cache-v2` at line 1 column 11" + )) + ); + } +} diff --git a/llama-cpp-bindings-build/src/cmake_replies.rs b/llama-cpp-bindings-build/src/cmake_replies.rs new file mode 100644 index 000000000..8e621a740 --- /dev/null +++ b/llama-cpp-bindings-build/src/cmake_replies.rs @@ -0,0 +1,10 @@ +use serde::Deserialize; + +use crate::cmake_reply_file::CmakeReplyFile; + +/// The replies `CMake` wrote for the shared stateless queries this crate submits. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +pub struct CmakeReplies { + #[serde(rename = "cache-v2")] + pub cache: CmakeReplyFile, +} diff --git a/llama-cpp-bindings-build/src/cmake_reply_file.rs b/llama-cpp-bindings-build/src/cmake_reply_file.rs new file mode 100644 index 000000000..86a0c7b7a --- /dev/null +++ b/llama-cpp-bindings-build/src/cmake_reply_file.rs @@ -0,0 +1,10 @@ +use std::path::PathBuf; + +use serde::Deserialize; + +/// A reference from the file API's reply index to the reply file holding one object. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +pub struct CmakeReplyFile { + #[serde(rename = "jsonFile")] + pub json_file: PathBuf, +} diff --git a/llama-cpp-bindings-build/src/cmake_reply_index.rs b/llama-cpp-bindings-build/src/cmake_reply_index.rs new file mode 100644 index 000000000..3d1bc3766 --- /dev/null +++ b/llama-cpp-bindings-build/src/cmake_reply_index.rs @@ -0,0 +1,9 @@ +use serde::Deserialize; + +use crate::cmake_replies::CmakeReplies; + +/// The file API's reply index, which names the reply file answering each query. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +pub struct CmakeReplyIndex { + pub reply: CmakeReplies, +} diff --git a/llama-cpp-bindings-build/src/lib.rs b/llama-cpp-bindings-build/src/lib.rs index 6ba978fd6..8f127e172 100644 --- a/llama-cpp-bindings-build/src/lib.rs +++ b/llama-cpp-bindings-build/src/lib.rs @@ -13,8 +13,14 @@ pub mod build_error; pub mod build_feature; mod cargo_directive; mod cmake_build; +mod cmake_cache; +mod cmake_cache_entry; mod cmake_configuration; mod cmake_define; +mod cmake_file_api; +mod cmake_replies; +mod cmake_reply_file; +mod cmake_reply_index; mod cpu_tuning; mod emit_directives; mod enabled_build_features; @@ -24,6 +30,8 @@ mod link_kind; mod native_build; mod native_library; mod native_profile; +mod openmp_linking; +mod openmp_runtime_source; mod privatized_field; mod rebuild_triggers; mod target_os; diff --git a/llama-cpp-bindings-build/src/link_directives.rs b/llama-cpp-bindings-build/src/link_directives.rs index c0b114114..47be2ee48 100644 --- a/llama-cpp-bindings-build/src/link_directives.rs +++ b/llama-cpp-bindings-build/src/link_directives.rs @@ -52,7 +52,7 @@ fn system_ggml_search_paths( } fn private_dependency_search_paths(context: &BuildContext) -> Vec { - let build_dir = context.cmake_dir().join("build"); + let build_dir = context.cmake_build_dir(); [ build_dir.join("common"), @@ -207,14 +207,6 @@ fn rocm_links( ]) } -fn openmp_links(context: &BuildContext, environment: &BuildEnvironment) -> Vec { - if environment.has(BuildFeature::Openmp) && context.cargo_cfg_target_env == "gnu" { - vec![link(LinkKind::Unspecified, "gomp")] - } else { - Vec::new() - } -} - fn platform_links(context: &BuildContext, environment: &BuildEnvironment) -> Vec { match context.target_os { TargetOs::Linux => vec![link(LinkKind::Dylib, "stdc++")], @@ -258,7 +250,6 @@ pub fn link_directives( native_library_links(context, environment), cuda_links(context, environment), rocm_links(context, environment)?, - openmp_links(context, environment), platform_links(context, environment), ] .concat()) @@ -567,25 +558,6 @@ mod tests { ); } - #[test] - fn openmp_links_libgomp_only_for_gnu_targets() { - assert_eq!( - ["gnu", "musl"].map(|cargo_cfg_target_env| { - platform_libraries( - BitFlags::from(BuildFeature::Openmp), - &[("CARGO_CFG_TARGET_ENV", cargo_cfg_target_env)], - ) - }), - [ - vec![ - library(LinkKind::Unspecified, "gomp"), - library(LinkKind::Dylib, "stdc++"), - ], - vec![library(LinkKind::Dylib, "stdc++")], - ] - ); - } - #[test] fn apple_and_android_targets_link_their_platform_cxx_runtime_and_metal_only_with_metal() { assert_eq!( diff --git a/llama-cpp-bindings-build/src/native_build.rs b/llama-cpp-bindings-build/src/native_build.rs index b342b68f7..40dd2ec7d 100644 --- a/llama-cpp-bindings-build/src/native_build.rs +++ b/llama-cpp-bindings-build/src/native_build.rs @@ -7,8 +7,10 @@ use crate::build_error::BuildError; use crate::cargo_directive::CargoDirective; use crate::cmake_build::CmakeBuild; use crate::cmake_configuration::CmakeConfiguration; +use crate::cmake_file_api::CmakeFileApi; use crate::emit_directives::emit_directives; use crate::link_directives::link_directives; +use crate::openmp_linking::OpenmpLinking; use crate::rebuild_triggers::rebuild_triggers; use crate::wrapper_compilation::WrapperCompilation; @@ -19,6 +21,7 @@ pub struct NativeBuild { pub wrappers: WrapperCompilation, pub cmake: CmakeBuild, pub directives: Vec, + pub openmp: OpenmpLinking, pub include_debug_messages: bool, } @@ -39,6 +42,9 @@ impl NativeBuild { cmake: CmakeBuild { source_dir: context.llama_src(), out_dir: context.cmake_dir(), + file_api: CmakeFileApi { + build_dir: context.cmake_build_dir(), + }, target_triple: context.target_triple.clone(), host_triple: context.host_triple.clone(), defines, @@ -51,12 +57,14 @@ impl NativeBuild { link_directives(context, environment)?, ] .concat(), + openmp: OpenmpLinking::of(context, environment), include_debug_messages: context.include_debug_messages, }) } /// Generates the bindings, compiles the wrappers, builds llama.cpp, and only then writes the - /// cargo directives to `output`. + /// cargo directives to `output`, with the OpenMP runtime `CMake` resolved linked after every + /// planned library. /// /// # Errors /// @@ -65,10 +73,12 @@ impl NativeBuild { let bindings_directives = self.bindings.generate()?; self.wrappers.compile()?; - self.cmake.build()?; + + let cmake_cache = self.cmake.build()?; + let openmp_directives = self.openmp.directives(&cmake_cache)?; emit_directives( - &[self.directives, bindings_directives].concat(), + &[self.directives, openmp_directives, bindings_directives].concat(), self.include_debug_messages, output, ) @@ -88,8 +98,12 @@ mod tests { use crate::cargo_directive::CargoDirective; use crate::cmake_build::CmakeBuild; use crate::cmake_define::CmakeDefine; + use crate::cmake_file_api::CmakeFileApi; use crate::link_kind::LinkKind; use crate::native_library::NativeLibrary; + use crate::openmp_linking::OpenmpLinking; + use crate::openmp_runtime_source::OpenmpRuntimeSource; + use crate::target_os::TargetOs; use crate::wrapper_compilation::WrapperCompilation; #[cfg(all(target_arch = "x86_64", target_os = "linux", target_env = "gnu"))] @@ -128,6 +142,9 @@ mod tests { cmake: CmakeBuild { source_dir: PathBuf::from(FIXTURES).join("cmake_project"), out_dir: root.join("cmake"), + file_api: CmakeFileApi { + build_dir: root.join("cmake/build"), + }, target_triple: HOST_TRIPLE.to_owned(), host_triple: HOST_TRIPLE.to_owned(), defines: vec![CmakeDefine::new("FIXTURE_LIBRARY_NAME", "fixture")], @@ -141,6 +158,10 @@ mod tests { }), CargoDirective::Debug("fixture planned".to_owned()), ], + openmp: OpenmpLinking { + source: OpenmpRuntimeSource::BundledGgml, + target_os: TargetOs::Linux, + }, include_debug_messages: false, } } @@ -170,6 +191,8 @@ mod tests { format!( "cargo:rustc-link-search=native={root}/cmake/lib\n\ cargo:rustc-link-lib=static=fixture\n\ + cargo:rustc-link-search=native={root}/cmake/lib\n\ + cargo:rustc-link-lib=dylib:+verbatim=libfixture.a\n\ cargo:rerun-if-env-changed=TARGET\n\ cargo:rerun-if-env-changed=BINDGEN_EXTRA_CLANG_ARGS\n\ cargo:rerun-if-changed={FIXTURES}/bindings/wrapper.h\n\ @@ -236,4 +259,52 @@ mod tests { assert_eq!(output, Vec::::new()); fs::remove_dir_all(root).unwrap(); } + + #[test] + fn a_build_tree_that_cannot_take_the_cache_query_stops_before_cmake_and_any_directive() { + let root = build_root("unqueryable-cmake"); + let mut native_build = fixture_build(&root); + let mut output = Vec::new(); + + native_build.cmake.file_api.build_dir = PathBuf::from(MANIFEST); + + assert_eq!( + native_build + .run(&mut output) + .err() + .map(|error| error.to_string()), + Some(format!( + "the CMake file API query could not be written into {MANIFEST}/.cmake/api/v1/\ + query: Not a directory (os error 20)" + )) + ); + assert_eq!( + [output.is_empty(), root.join("cmake/lib").exists()], + [true, false] + ); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn an_apple_build_whose_ggml_needs_an_openmp_runtime_emits_no_directive() { + let root = build_root("apple-openmp"); + let mut native_build = fixture_build(&root); + let mut output = Vec::new(); + + native_build.openmp.target_os = TargetOs::Apple; + + assert_eq!( + native_build + .run(&mut output) + .err() + .map(|error| error.to_string()), + Some(format!( + "ggml needs the OpenMP runtime [{:?}], which Apple's linker cannot link by its \ + exact file name", + root.join("cmake/lib/libfixture.a") + )) + ); + assert_eq!(output, Vec::::new()); + fs::remove_dir_all(root).unwrap(); + } } diff --git a/llama-cpp-bindings-build/src/openmp_linking.rs b/llama-cpp-bindings-build/src/openmp_linking.rs new file mode 100644 index 000000000..c5d69f2e9 --- /dev/null +++ b/llama-cpp-bindings-build/src/openmp_linking.rs @@ -0,0 +1,120 @@ +use std::path::PathBuf; + +use crate::build_context::BuildContext; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::cargo_directive::CargoDirective; +use crate::cmake_cache::CmakeCache; +use crate::openmp_runtime_source::OpenmpRuntimeSource; +use crate::target_os::TargetOs; + +fn link_by_file_name(library: PathBuf) -> Result<[CargoDirective; 2], BuildError> { + match (library.parent(), library.file_name()) { + (Some(directory), Some(file_name)) => Ok([ + CargoDirective::LinkSearchNative(directory.to_path_buf()), + CargoDirective::LinkDylibByFileName(PathBuf::from(file_name)), + ]), + _ => Err(BuildError::OpenmpRuntimeLibraryWithoutFileName { library }), + } +} + +/// Links the OpenMP runtime `CMake` resolved for ggml, which is only known once `CMake` has +/// configured the build. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct OpenmpLinking { + pub source: OpenmpRuntimeSource, + pub target_os: TargetOs, +} + +impl OpenmpLinking { + #[must_use] + pub fn of(context: &BuildContext, environment: &BuildEnvironment) -> Self { + Self { + source: OpenmpRuntimeSource::of(context, environment), + target_os: context.target_os.clone(), + } + } + + /// Links each runtime library by its exact file name from its own directory, after every + /// library that needs it. + /// + /// # Errors + /// + /// Returns the [`BuildError`] of [`OpenmpRuntimeSource::libraries`], + /// [`BuildError::OpenmpRuntimeUnlinkableOnApple`] when an Apple target needs a runtime, or + /// [`BuildError::OpenmpRuntimeLibraryWithoutFileName`] when a runtime library path names no + /// file. + pub fn directives(&self, cache: &CmakeCache) -> Result, BuildError> { + let libraries = self.source.libraries(cache)?; + + if !libraries.is_empty() && matches!(self.target_os, TargetOs::Apple) { + return Err(BuildError::OpenmpRuntimeUnlinkableOnApple { libraries }); + } + + libraries + .into_iter() + .map(link_by_file_name) + .collect::, BuildError>>() + .map(|directives| directives.into_iter().flatten().collect()) + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::OpenmpLinking; + use crate::cargo_directive::CargoDirective; + use crate::cmake_cache::tests::cache_of; + use crate::openmp_runtime_source::OpenmpRuntimeSource; + use crate::target_os::TargetOs; + + fn linux_directives(entries: &[(&str, &str)]) -> Result, String> { + OpenmpLinking { + source: OpenmpRuntimeSource::BundledGgml, + target_os: TargetOs::Linux, + } + .directives(&cache_of(entries)) + .map_err(|error| error.to_string()) + } + + #[test] + fn each_runtime_library_is_linked_by_its_exact_file_name_from_its_directory() { + assert_eq!( + linux_directives(&[ + ("GGML_OPENMP_ENABLED", "ON"), + ("OpenMP_C_LIB_NAMES", "gomp;pthread"), + ("OpenMP_CXX_LIB_NAMES", "gomp;pthread"), + ("OpenMP_gomp_LIBRARY", "/usr/lib/gcc/libgomp.so"), + ("OpenMP_pthread_LIBRARY", "/usr/lib/libpthread.a"), + ]), + Ok(vec![ + CargoDirective::LinkSearchNative(PathBuf::from("/usr/lib/gcc")), + CargoDirective::LinkDylibByFileName(PathBuf::from("libgomp.so")), + CargoDirective::LinkSearchNative(PathBuf::from("/usr/lib")), + CargoDirective::LinkDylibByFileName(PathBuf::from("libpthread.a")), + ]) + ); + } + + #[test] + fn a_cache_without_the_bundled_ggml_openmp_switch_is_refused() { + assert_eq!( + linux_directives(&[]), + Err("the CMake cache has no entry GGML_OPENMP_ENABLED".to_owned()) + ); + } + + #[test] + fn a_runtime_library_path_without_a_file_name_is_refused() { + assert_eq!( + linux_directives(&[ + ("GGML_OPENMP_ENABLED", "ON"), + ("OpenMP_C_LIB_NAMES", "gomp"), + ("OpenMP_CXX_LIB_NAMES", "gomp"), + ("OpenMP_gomp_LIBRARY", "/"), + ]), + Err("the OpenMP runtime library / names no file".to_owned()) + ); + } +} diff --git a/llama-cpp-bindings-build/src/openmp_runtime_source.rs b/llama-cpp-bindings-build/src/openmp_runtime_source.rs new file mode 100644 index 000000000..9281abbb8 --- /dev/null +++ b/llama-cpp-bindings-build/src/openmp_runtime_source.rs @@ -0,0 +1,235 @@ +use std::path::PathBuf; + +use crate::build_context::BuildContext; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::build_feature::BuildFeature; +use crate::cmake_cache::CmakeCache; + +const BUNDLED_GGML_OPENMP_SWITCH: &str = "GGML_OPENMP_ENABLED"; + +const C_RUNTIME_LIBRARY_NAMES: &str = "OpenMP_C_LIB_NAMES"; + +const CXX_RUNTIME_LIBRARY_NAMES: &str = "OpenMP_CXX_LIB_NAMES"; + +fn bundled_ggml_uses_openmp(cache: &CmakeCache) -> Result { + match cache.value(BUNDLED_GGML_OPENMP_SWITCH)? { + "ON" => Ok(true), + "OFF" => Ok(false), + value => Err(BuildError::CmakeCacheEntryNotASwitch { + name: BUNDLED_GGML_OPENMP_SWITCH, + value: value.to_owned(), + }), + } +} + +fn runtime_library_names(cache: &CmakeCache) -> Result, BuildError> { + let mut names = Vec::new(); + + for library_names in [C_RUNTIME_LIBRARY_NAMES, CXX_RUNTIME_LIBRARY_NAMES] { + for name in cache + .value(library_names)? + .split(';') + .filter(|name| !name.is_empty()) + { + if !names.contains(&name) { + names.push(name); + } + } + } + + Ok(names) +} + +/// Where the build learns which OpenMP runtime libraries a statically linked ggml needs. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum OpenmpRuntimeSource { + SharedLibraries, + BundledGgml, + SystemGgml, +} + +impl OpenmpRuntimeSource { + #[must_use] + pub fn of(context: &BuildContext, environment: &BuildEnvironment) -> Self { + match ( + context.build_shared_libs, + environment.has(BuildFeature::SystemGgml), + environment.has(BuildFeature::SystemGgmlStatic), + ) { + (true, _, _) | (false, true, false) => Self::SharedLibraries, + (false, true, true) => Self::SystemGgml, + (false, false, _) => Self::BundledGgml, + } + } + + /// The OpenMP runtime libraries `CMake` resolved for ggml, which only a statically linked + /// ggml leaves for the bindings to link. + /// + /// # Errors + /// + /// Returns [`BuildError::CmakeCacheEntryMissing`] when the cache lacks an entry that tells + /// whether or where ggml's OpenMP runtime is, or [`BuildError::CmakeCacheEntryNotASwitch`] + /// when the bundled ggml's OpenMP switch is neither `ON` nor `OFF`. + pub fn libraries(self, cache: &CmakeCache) -> Result, BuildError> { + let uses_openmp = match self { + Self::SharedLibraries => false, + Self::BundledGgml => bundled_ggml_uses_openmp(cache)?, + Self::SystemGgml => cache.contains(C_RUNTIME_LIBRARY_NAMES), + }; + + if !uses_openmp { + return Ok(Vec::new()); + } + + runtime_library_names(cache)? + .into_iter() + .map(|name| { + cache + .value(&format!("OpenMP_{name}_LIBRARY")) + .map(PathBuf::from) + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use enumflags2::BitFlags; + + use super::OpenmpRuntimeSource; + use crate::build_context::tests::cargo_environment; + use crate::build_context::tests::context_of; + use crate::build_feature::BuildFeature; + use crate::cmake_cache::tests::cache_of; + + const GNU_RUNTIME: [(&str, &str); 4] = [ + ("OpenMP_C_LIB_NAMES", "gomp;pthread"), + ("OpenMP_CXX_LIB_NAMES", "gomp"), + ("OpenMP_gomp_LIBRARY", "/usr/lib/gcc/libgomp.so"), + ("OpenMP_pthread_LIBRARY", "/usr/lib/libpthread.a"), + ]; + + fn gnu_runtime_with(entry: (&'static str, &'static str)) -> Vec<(&'static str, &'static str)> { + [&[entry][..], &GNU_RUNTIME].concat() + } + + fn failure(source: OpenmpRuntimeSource, entries: &[(&str, &str)]) -> Option { + source + .libraries(&cache_of(entries)) + .err() + .map(|error| error.to_string()) + } + + #[test] + fn only_a_statically_linked_ggml_leaves_its_runtime_to_the_bindings() { + assert_eq!( + [ + BitFlags::from(BuildFeature::DynamicLink), + BitFlags::from(BuildFeature::SystemGgml), + BuildFeature::SystemGgml | BuildFeature::SystemGgmlStatic, + BitFlags::empty(), + ] + .map(|features| { + let environment = cargo_environment(features, &[]); + + OpenmpRuntimeSource::of(&context_of(&environment), &environment) + }), + [ + OpenmpRuntimeSource::SharedLibraries, + OpenmpRuntimeSource::SharedLibraries, + OpenmpRuntimeSource::SystemGgml, + OpenmpRuntimeSource::BundledGgml, + ] + ); + } + + #[test] + fn shared_libraries_need_no_runtime_from_the_cache() { + assert_eq!( + OpenmpRuntimeSource::SharedLibraries + .libraries(&cache_of(&[])) + .unwrap(), + Vec::::new() + ); + } + + #[test] + fn a_bundled_ggml_with_openmp_needs_every_runtime_library_of_both_languages_once() { + assert_eq!( + OpenmpRuntimeSource::BundledGgml + .libraries(&cache_of(&gnu_runtime_with(("GGML_OPENMP_ENABLED", "ON")))) + .unwrap(), + [ + PathBuf::from("/usr/lib/gcc/libgomp.so"), + PathBuf::from("/usr/lib/libpthread.a") + ] + ); + } + + #[test] + fn a_bundled_ggml_without_openmp_needs_no_runtime() { + assert_eq!( + OpenmpRuntimeSource::BundledGgml + .libraries(&cache_of(&gnu_runtime_with(("GGML_OPENMP_ENABLED", "OFF")))) + .unwrap(), + Vec::::new() + ); + } + + #[test] + fn a_bundled_ggml_openmp_switch_other_than_on_or_off_is_refused() { + assert_eq!( + failure( + OpenmpRuntimeSource::BundledGgml, + &gnu_runtime_with(("GGML_OPENMP_ENABLED", "TRUE")) + ), + Some( + "the CMake cache entry GGML_OPENMP_ENABLED is TRUE, neither ON nor OFF".to_owned() + ) + ); + } + + #[test] + fn a_system_ggml_needs_the_runtime_only_when_its_config_searched_for_openmp() { + assert_eq!( + [cache_of(&[]), cache_of(&GNU_RUNTIME)] + .map(|cache| OpenmpRuntimeSource::SystemGgml.libraries(&cache).unwrap()), + [ + Vec::new(), + vec![ + PathBuf::from("/usr/lib/gcc/libgomp.so"), + PathBuf::from("/usr/lib/libpthread.a") + ] + ] + ); + } + + #[test] + fn a_bundled_ggml_with_openmp_but_no_runtime_library_list_is_refused() { + assert_eq!( + failure( + OpenmpRuntimeSource::BundledGgml, + &[("GGML_OPENMP_ENABLED", "ON")] + ), + Some("the CMake cache has no entry OpenMP_C_LIB_NAMES".to_owned()) + ); + } + + #[test] + fn a_runtime_library_cmake_did_not_locate_is_refused() { + assert_eq!( + failure( + OpenmpRuntimeSource::BundledGgml, + &[ + ("GGML_OPENMP_ENABLED", "ON"), + ("OpenMP_C_LIB_NAMES", "gomp"), + ("OpenMP_CXX_LIB_NAMES", "gomp"), + ] + ), + Some("the CMake cache has no entry OpenMP_gomp_LIBRARY".to_owned()) + ); + } +} From 97b85100489984c82109c4a2ecc69886667c74fb Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 16:05:59 +0100 Subject: [PATCH 77/82] Query the CMake cache from an initial-cache script so a wiped build tree still reports it --- .../cmake/cache_query.cmake | 1 + llama-cpp-bindings-build/src/build_error.rs | 6 -- llama-cpp-bindings-build/src/cmake_build.rs | 6 +- .../src/cmake_file_api.rs | 27 ++----- llama-cpp-bindings-build/src/native_build.rs | 79 ++++++++++--------- 5 files changed, 53 insertions(+), 66 deletions(-) create mode 100644 llama-cpp-bindings-build/cmake/cache_query.cmake diff --git a/llama-cpp-bindings-build/cmake/cache_query.cmake b/llama-cpp-bindings-build/cmake/cache_query.cmake new file mode 100644 index 000000000..590718d7d --- /dev/null +++ b/llama-cpp-bindings-build/cmake/cache_query.cmake @@ -0,0 +1 @@ +file(WRITE "${CMAKE_BINARY_DIR}/.cmake/api/v1/query/cache-v2" "") diff --git a/llama-cpp-bindings-build/src/build_error.rs b/llama-cpp-bindings-build/src/build_error.rs index 9aa6d3c97..2ec0b93c7 100644 --- a/llama-cpp-bindings-build/src/build_error.rs +++ b/llama-cpp-bindings-build/src/build_error.rs @@ -73,12 +73,6 @@ pub enum BuildError { EnvironmentDirectory { name: &'static str, path: PathBuf }, #[error("cargo directives could not be written: {0}")] DirectiveWrite(#[source] io::Error), - #[error("the CMake file API query could not be written into {path}: {source}")] - CmakeFileApiQueryNotWritten { - path: PathBuf, - #[source] - source: io::Error, - }, #[error("the CMake file API reply directory {path} could not be listed: {source}")] CmakeFileApiReplyDirectoryUnreadable { path: PathBuf, diff --git a/llama-cpp-bindings-build/src/cmake_build.rs b/llama-cpp-bindings-build/src/cmake_build.rs index 5e914e213..a039f9f59 100644 --- a/llama-cpp-bindings-build/src/cmake_build.rs +++ b/llama-cpp-bindings-build/src/cmake_build.rs @@ -26,8 +26,7 @@ impl CmakeBuild { /// # Errors /// /// Returns [`BuildError::Filesystem`] when a directory the project installs into cannot be - /// created, or the [`BuildError`] of [`CmakeFileApi::request_cache`] or - /// [`CmakeFileApi::cache`]. + /// created, or the [`BuildError`] of [`CmakeFileApi::cache`]. /// /// # Panics /// @@ -39,7 +38,6 @@ impl CmakeBuild { source, }) })?; - self.file_api.request_cache()?; let mut config = Config::new(&self.source_dir); @@ -48,6 +46,8 @@ impl CmakeBuild { } config + .configure_arg("-C") + .configure_arg(CmakeFileApi::CACHE_QUERY_SCRIPT) .target(&self.target_triple) .host(&self.host_triple) .out_dir(&self.out_dir) diff --git a/llama-cpp-bindings-build/src/cmake_file_api.rs b/llama-cpp-bindings-build/src/cmake_file_api.rs index b96e4e4fd..e0fa22ad7 100644 --- a/llama-cpp-bindings-build/src/cmake_file_api.rs +++ b/llama-cpp-bindings-build/src/cmake_file_api.rs @@ -48,27 +48,12 @@ pub struct CmakeFileApi { } impl CmakeFileApi { - fn api_dir(&self) -> PathBuf { - self.build_dir.join(".cmake").join("api").join("v1") - } - - /// Asks `CMake` to answer with its cache the next time it configures this build tree. - /// - /// # Errors - /// - /// Returns [`BuildError::CmakeFileApiQueryNotWritten`] when the query cannot be written. - pub fn request_cache(&self) -> Result<(), BuildError> { - let query_dir = self.api_dir().join("query"); - - fs::create_dir_all(&query_dir) - .and_then(|()| fs::write(query_dir.join("cache-v2"), b"")) - .map_err(|source| BuildError::CmakeFileApiQueryNotWritten { - path: query_dir, - source, - }) - } + /// The initial-cache script that, passed to `CMake` with `-C`, has every configure of the + /// build tree query its cache before `CMake` reads the queries. + pub const CACHE_QUERY_SCRIPT: &'static str = + concat!(env!("CARGO_MANIFEST_DIR"), "/cmake/cache_query.cmake"); - /// The cache `CMake` reported in its current reply to [`Self::request_cache`]. + /// The cache `CMake` reported in its current reply to [`Self::CACHE_QUERY_SCRIPT`]. /// /// # Errors /// @@ -78,7 +63,7 @@ impl CmakeFileApi { /// [`BuildError::CmakeFileApiReplyFileInvalid`] when a reply file cannot be read as the /// reply this crate queried. pub fn cache(&self) -> Result { - let reply_dir = self.api_dir().join("reply"); + let reply_dir = self.build_dir.join(".cmake/api/v1/reply"); let CmakeReplyIndex { reply } = read_reply_file(current_reply_index(&reply_dir)?)?; read_reply_file(reply_dir.join(reply.cache.json_file)) diff --git a/llama-cpp-bindings-build/src/native_build.rs b/llama-cpp-bindings-build/src/native_build.rs index 40dd2ec7d..650ac725d 100644 --- a/llama-cpp-bindings-build/src/native_build.rs +++ b/llama-cpp-bindings-build/src/native_build.rs @@ -166,6 +166,20 @@ mod tests { } } + fn successful_directives(root: &Path) -> String { + format!( + "cargo:rustc-link-search=native={root}/cmake/lib\n\ + cargo:rustc-link-lib=static=fixture\n\ + cargo:rustc-link-search=native={root}/cmake/lib\n\ + cargo:rustc-link-lib=dylib:+verbatim=libfixture.a\n\ + cargo:rerun-if-env-changed=TARGET\n\ + cargo:rerun-if-env-changed=BINDGEN_EXTRA_CLANG_ARGS\n\ + cargo:rerun-if-changed={FIXTURES}/bindings/wrapper.h\n\ + cargo:rerun-if-changed={FIXTURES}/bindings/llama_types.h\n", + root = root.display() + ) + } + fn build_root(name: &str) -> PathBuf { let root = env::temp_dir().join(format!("llama-cpp-bindings-build-{name}-{}", process::id())); @@ -188,17 +202,7 @@ mod tests { assert_eq!( String::from_utf8(output).unwrap(), - format!( - "cargo:rustc-link-search=native={root}/cmake/lib\n\ - cargo:rustc-link-lib=static=fixture\n\ - cargo:rustc-link-search=native={root}/cmake/lib\n\ - cargo:rustc-link-lib=dylib:+verbatim=libfixture.a\n\ - cargo:rerun-if-env-changed=TARGET\n\ - cargo:rerun-if-env-changed=BINDGEN_EXTRA_CLANG_ARGS\n\ - cargo:rerun-if-changed={FIXTURES}/bindings/wrapper.h\n\ - cargo:rerun-if-changed={FIXTURES}/bindings/llama_types.h\n", - root = root.display() - ) + successful_directives(&root) ); assert_eq!( [ @@ -214,6 +218,34 @@ mod tests { } } + #[test] + fn a_build_tree_left_by_a_relocated_source_still_reports_the_cmake_cache() { + let root = build_root("relocated-source"); + let relocated_source = root.join("relocated_cmake_project"); + let mut native_build = fixture_build(&root); + let mut output = Vec::new(); + + fixture_build(&root).run(&mut Vec::new()).unwrap(); + fs::create_dir_all(&relocated_source).unwrap(); + for file_name in ["CMakeLists.txt", "fixture.c"] { + fs::copy( + PathBuf::from(FIXTURES) + .join("cmake_project") + .join(file_name), + relocated_source.join(file_name), + ) + .unwrap(); + } + native_build.cmake.source_dir = relocated_source; + native_build.run(&mut output).unwrap(); + + assert_eq!( + String::from_utf8(output).unwrap(), + successful_directives(&root) + ); + fs::remove_dir_all(root).unwrap(); + } + #[test] fn a_failed_wrapper_compilation_stops_the_build_before_cmake_and_any_directive() { let root = build_root("failed-wrappers"); @@ -260,31 +292,6 @@ mod tests { fs::remove_dir_all(root).unwrap(); } - #[test] - fn a_build_tree_that_cannot_take_the_cache_query_stops_before_cmake_and_any_directive() { - let root = build_root("unqueryable-cmake"); - let mut native_build = fixture_build(&root); - let mut output = Vec::new(); - - native_build.cmake.file_api.build_dir = PathBuf::from(MANIFEST); - - assert_eq!( - native_build - .run(&mut output) - .err() - .map(|error| error.to_string()), - Some(format!( - "the CMake file API query could not be written into {MANIFEST}/.cmake/api/v1/\ - query: Not a directory (os error 20)" - )) - ); - assert_eq!( - [output.is_empty(), root.join("cmake/lib").exists()], - [true, false] - ); - fs::remove_dir_all(root).unwrap(); - } - #[test] fn an_apple_build_whose_ggml_needs_an_openmp_runtime_emits_no_directive() { let root = build_root("apple-openmp"); From 5103e8ebfc44aa99ac186e3b5d704465abfb66f5 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 16:27:08 +0100 Subject: [PATCH 78/82] Compare a CPU-only load key with one requesting every GPU layer as llama.cpp defines it --- llama-cpp-test-harness/src/load_key.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama-cpp-test-harness/src/load_key.rs b/llama-cpp-test-harness/src/load_key.rs index 049358f2c..2ae9f9ff0 100644 --- a/llama-cpp-test-harness/src/load_key.rs +++ b/llama-cpp-test-harness/src/load_key.rs @@ -88,7 +88,7 @@ mod tests { #[test] fn different_model_load_params_compare_unequal() { let mut other = baseline(); - other.model_load_params.n_gpu_layers = 999; + other.model_load_params.n_gpu_layers = -1; assert_ne!(baseline(), other); } From 51463a6e3d7c7f34a550b7d2d5c07a70a829d5d5 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 16:51:54 +0100 Subject: [PATCH 79/82] Read CMake switches and lists on the cache itself and build test caches from entries --- llama-cpp-bindings-build/src/build_error.rs | 2 +- llama-cpp-bindings-build/src/cmake_cache.rs | 71 +++++++- .../src/openmp_linking.rs | 32 ++-- .../src/openmp_runtime_source.rs | 159 ++++++++---------- 4 files changed, 146 insertions(+), 118 deletions(-) diff --git a/llama-cpp-bindings-build/src/build_error.rs b/llama-cpp-bindings-build/src/build_error.rs index 2ec0b93c7..9aa224e80 100644 --- a/llama-cpp-bindings-build/src/build_error.rs +++ b/llama-cpp-bindings-build/src/build_error.rs @@ -96,7 +96,7 @@ pub enum BuildError { #[error("the CMake cache has no entry {name}")] CmakeCacheEntryMissing { name: String }, #[error("the CMake cache entry {name} is {value}, neither ON nor OFF")] - CmakeCacheEntryNotASwitch { name: &'static str, value: String }, + CmakeCacheEntryNotASwitch { name: String, value: String }, #[error( "ggml needs the OpenMP runtime {libraries:?}, which Apple's linker cannot link by its \ exact file name" diff --git a/llama-cpp-bindings-build/src/cmake_cache.rs b/llama-cpp-bindings-build/src/cmake_cache.rs index f3c91306b..a126495ad 100644 --- a/llama-cpp-bindings-build/src/cmake_cache.rs +++ b/llama-cpp-bindings-build/src/cmake_cache.rs @@ -27,6 +27,37 @@ impl CmakeCache { name: name.to_owned(), }) } + + /// The switch `name`, which the project writes into the cache as `ON` or `OFF`. + /// + /// # Errors + /// + /// Returns [`BuildError::CmakeCacheEntryMissing`] when the cache has no entry `name`, or + /// [`BuildError::CmakeCacheEntryNotASwitch`] when it holds anything but `ON` or `OFF`. + pub fn switch(&self, name: &str) -> Result { + match self.value(name)? { + "ON" => Ok(true), + "OFF" => Ok(false), + value => Err(BuildError::CmakeCacheEntryNotASwitch { + name: name.to_owned(), + value: value.to_owned(), + }), + } + } + + /// The elements of the `CMake` list `name`, which `CMake` separates with semicolons and + /// writes as an empty value when the list has none. + /// + /// # Errors + /// + /// Returns [`BuildError::CmakeCacheEntryMissing`] when the cache has no entry `name`. + pub fn list(&self, name: &str) -> Result, BuildError> { + Ok(self + .value(name)? + .split(';') + .filter(|element| !element.is_empty()) + .collect()) + } } #[cfg(test)] @@ -34,15 +65,37 @@ pub mod tests { use super::CmakeCache; use crate::cmake_cache_entry::CmakeCacheEntry; - pub fn cache_of(entries: &[(&str, &str)]) -> CmakeCache { - CmakeCache { - entries: entries - .iter() - .map(|(name, value)| CmakeCacheEntry { - name: (*name).to_owned(), - value: (*value).to_owned(), - }) - .collect(), + pub fn entry(name: &str, value: &str) -> CmakeCacheEntry { + CmakeCacheEntry { + name: name.to_owned(), + value: value.to_owned(), } } + + #[test] + fn a_switch_written_as_anything_but_on_or_off_is_refused() { + assert_eq!( + CmakeCache { + entries: vec![entry("GGML_OPENMP_ENABLED", "TRUE")], + } + .switch("GGML_OPENMP_ENABLED") + .err() + .map(|error| error.to_string()), + Some( + "the CMake cache entry GGML_OPENMP_ENABLED is TRUE, neither ON nor OFF".to_owned() + ) + ); + } + + #[test] + fn an_empty_list_has_no_elements() { + assert_eq!( + CmakeCache { + entries: vec![entry("OpenMP_C_LIB_NAMES", "")], + } + .list("OpenMP_C_LIB_NAMES") + .unwrap(), + Vec::<&str>::new() + ); + } } diff --git a/llama-cpp-bindings-build/src/openmp_linking.rs b/llama-cpp-bindings-build/src/openmp_linking.rs index c5d69f2e9..6a2d0c10f 100644 --- a/llama-cpp-bindings-build/src/openmp_linking.rs +++ b/llama-cpp-bindings-build/src/openmp_linking.rs @@ -65,28 +65,30 @@ mod tests { use super::OpenmpLinking; use crate::cargo_directive::CargoDirective; - use crate::cmake_cache::tests::cache_of; + use crate::cmake_cache::CmakeCache; + use crate::cmake_cache::tests::entry; + use crate::cmake_cache_entry::CmakeCacheEntry; use crate::openmp_runtime_source::OpenmpRuntimeSource; use crate::target_os::TargetOs; - fn linux_directives(entries: &[(&str, &str)]) -> Result, String> { + fn linux_directives(entries: Vec) -> Result, String> { OpenmpLinking { source: OpenmpRuntimeSource::BundledGgml, target_os: TargetOs::Linux, } - .directives(&cache_of(entries)) + .directives(&CmakeCache { entries }) .map_err(|error| error.to_string()) } #[test] fn each_runtime_library_is_linked_by_its_exact_file_name_from_its_directory() { assert_eq!( - linux_directives(&[ - ("GGML_OPENMP_ENABLED", "ON"), - ("OpenMP_C_LIB_NAMES", "gomp;pthread"), - ("OpenMP_CXX_LIB_NAMES", "gomp;pthread"), - ("OpenMP_gomp_LIBRARY", "/usr/lib/gcc/libgomp.so"), - ("OpenMP_pthread_LIBRARY", "/usr/lib/libpthread.a"), + linux_directives(vec![ + entry("GGML_OPENMP_ENABLED", "ON"), + entry("OpenMP_C_LIB_NAMES", "gomp;pthread"), + entry("OpenMP_CXX_LIB_NAMES", "gomp;pthread"), + entry("OpenMP_gomp_LIBRARY", "/usr/lib/gcc/libgomp.so"), + entry("OpenMP_pthread_LIBRARY", "/usr/lib/libpthread.a"), ]), Ok(vec![ CargoDirective::LinkSearchNative(PathBuf::from("/usr/lib/gcc")), @@ -100,7 +102,7 @@ mod tests { #[test] fn a_cache_without_the_bundled_ggml_openmp_switch_is_refused() { assert_eq!( - linux_directives(&[]), + linux_directives(Vec::new()), Err("the CMake cache has no entry GGML_OPENMP_ENABLED".to_owned()) ); } @@ -108,11 +110,11 @@ mod tests { #[test] fn a_runtime_library_path_without_a_file_name_is_refused() { assert_eq!( - linux_directives(&[ - ("GGML_OPENMP_ENABLED", "ON"), - ("OpenMP_C_LIB_NAMES", "gomp"), - ("OpenMP_CXX_LIB_NAMES", "gomp"), - ("OpenMP_gomp_LIBRARY", "/"), + linux_directives(vec![ + entry("GGML_OPENMP_ENABLED", "ON"), + entry("OpenMP_C_LIB_NAMES", "gomp"), + entry("OpenMP_CXX_LIB_NAMES", "gomp"), + entry("OpenMP_gomp_LIBRARY", "/"), ]), Err("the OpenMP runtime library / names no file".to_owned()) ); diff --git a/llama-cpp-bindings-build/src/openmp_runtime_source.rs b/llama-cpp-bindings-build/src/openmp_runtime_source.rs index 9281abbb8..5a808fadc 100644 --- a/llama-cpp-bindings-build/src/openmp_runtime_source.rs +++ b/llama-cpp-bindings-build/src/openmp_runtime_source.rs @@ -12,35 +12,6 @@ const C_RUNTIME_LIBRARY_NAMES: &str = "OpenMP_C_LIB_NAMES"; const CXX_RUNTIME_LIBRARY_NAMES: &str = "OpenMP_CXX_LIB_NAMES"; -fn bundled_ggml_uses_openmp(cache: &CmakeCache) -> Result { - match cache.value(BUNDLED_GGML_OPENMP_SWITCH)? { - "ON" => Ok(true), - "OFF" => Ok(false), - value => Err(BuildError::CmakeCacheEntryNotASwitch { - name: BUNDLED_GGML_OPENMP_SWITCH, - value: value.to_owned(), - }), - } -} - -fn runtime_library_names(cache: &CmakeCache) -> Result, BuildError> { - let mut names = Vec::new(); - - for library_names in [C_RUNTIME_LIBRARY_NAMES, CXX_RUNTIME_LIBRARY_NAMES] { - for name in cache - .value(library_names)? - .split(';') - .filter(|name| !name.is_empty()) - { - if !names.contains(&name) { - names.push(name); - } - } - } - - Ok(names) -} - /// Where the build learns which OpenMP runtime libraries a statically linked ggml needs. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum OpenmpRuntimeSource { @@ -63,26 +34,38 @@ impl OpenmpRuntimeSource { } } + fn needs_runtime(self, cache: &CmakeCache) -> Result { + match self { + Self::SharedLibraries => Ok(false), + Self::BundledGgml => cache.switch(BUNDLED_GGML_OPENMP_SWITCH), + Self::SystemGgml => Ok(cache.contains(C_RUNTIME_LIBRARY_NAMES)), + } + } + /// The OpenMP runtime libraries `CMake` resolved for ggml, which only a statically linked /// ggml leaves for the bindings to link. /// /// # Errors /// - /// Returns [`BuildError::CmakeCacheEntryMissing`] when the cache lacks an entry that tells - /// whether or where ggml's OpenMP runtime is, or [`BuildError::CmakeCacheEntryNotASwitch`] - /// when the bundled ggml's OpenMP switch is neither `ON` nor `OFF`. + /// Returns the [`BuildError`] of [`CmakeCache::switch`], [`CmakeCache::list`], or + /// [`CmakeCache::value`] when the cache does not tell whether or where ggml's OpenMP + /// runtime is. pub fn libraries(self, cache: &CmakeCache) -> Result, BuildError> { - let uses_openmp = match self { - Self::SharedLibraries => false, - Self::BundledGgml => bundled_ggml_uses_openmp(cache)?, - Self::SystemGgml => cache.contains(C_RUNTIME_LIBRARY_NAMES), - }; - - if !uses_openmp { + if !self.needs_runtime(cache)? { return Ok(Vec::new()); } - runtime_library_names(cache)? + let mut names = Vec::new(); + + for library_names in [C_RUNTIME_LIBRARY_NAMES, CXX_RUNTIME_LIBRARY_NAMES] { + for name in cache.list(library_names)? { + if !names.contains(&name) { + names.push(name); + } + } + } + + names .into_iter() .map(|name| { cache @@ -103,22 +86,35 @@ mod tests { use crate::build_context::tests::cargo_environment; use crate::build_context::tests::context_of; use crate::build_feature::BuildFeature; - use crate::cmake_cache::tests::cache_of; + use crate::cmake_cache::CmakeCache; + use crate::cmake_cache::tests::entry; + use crate::cmake_cache_entry::CmakeCacheEntry; + + fn gnu_runtime() -> Vec { + vec![ + entry("OpenMP_C_LIB_NAMES", "gomp;pthread"), + entry("OpenMP_CXX_LIB_NAMES", "gomp"), + entry("OpenMP_gomp_LIBRARY", "/usr/lib/gcc/libgomp.so"), + entry("OpenMP_pthread_LIBRARY", "/usr/lib/libpthread.a"), + ] + } - const GNU_RUNTIME: [(&str, &str); 4] = [ - ("OpenMP_C_LIB_NAMES", "gomp;pthread"), - ("OpenMP_CXX_LIB_NAMES", "gomp"), - ("OpenMP_gomp_LIBRARY", "/usr/lib/gcc/libgomp.so"), - ("OpenMP_pthread_LIBRARY", "/usr/lib/libpthread.a"), - ]; + fn bundled_ggml_with_openmp(switch: &str) -> CmakeCache { + CmakeCache { + entries: [vec![entry("GGML_OPENMP_ENABLED", switch)], gnu_runtime()].concat(), + } + } - fn gnu_runtime_with(entry: (&'static str, &'static str)) -> Vec<(&'static str, &'static str)> { - [&[entry][..], &GNU_RUNTIME].concat() + fn gnu_runtime_libraries() -> Vec { + vec![ + PathBuf::from("/usr/lib/gcc/libgomp.so"), + PathBuf::from("/usr/lib/libpthread.a"), + ] } - fn failure(source: OpenmpRuntimeSource, entries: &[(&str, &str)]) -> Option { - source - .libraries(&cache_of(entries)) + fn bundled_ggml_failure(entries: Vec) -> Option { + OpenmpRuntimeSource::BundledGgml + .libraries(&CmakeCache { entries }) .err() .map(|error| error.to_string()) } @@ -150,7 +146,9 @@ mod tests { fn shared_libraries_need_no_runtime_from_the_cache() { assert_eq!( OpenmpRuntimeSource::SharedLibraries - .libraries(&cache_of(&[])) + .libraries(&CmakeCache { + entries: Vec::new() + }) .unwrap(), Vec::::new() ); @@ -160,12 +158,9 @@ mod tests { fn a_bundled_ggml_with_openmp_needs_every_runtime_library_of_both_languages_once() { assert_eq!( OpenmpRuntimeSource::BundledGgml - .libraries(&cache_of(&gnu_runtime_with(("GGML_OPENMP_ENABLED", "ON")))) + .libraries(&bundled_ggml_with_openmp("ON")) .unwrap(), - [ - PathBuf::from("/usr/lib/gcc/libgomp.so"), - PathBuf::from("/usr/lib/libpthread.a") - ] + gnu_runtime_libraries() ); } @@ -173,47 +168,28 @@ mod tests { fn a_bundled_ggml_without_openmp_needs_no_runtime() { assert_eq!( OpenmpRuntimeSource::BundledGgml - .libraries(&cache_of(&gnu_runtime_with(("GGML_OPENMP_ENABLED", "OFF")))) + .libraries(&bundled_ggml_with_openmp("OFF")) .unwrap(), Vec::::new() ); } - #[test] - fn a_bundled_ggml_openmp_switch_other_than_on_or_off_is_refused() { - assert_eq!( - failure( - OpenmpRuntimeSource::BundledGgml, - &gnu_runtime_with(("GGML_OPENMP_ENABLED", "TRUE")) - ), - Some( - "the CMake cache entry GGML_OPENMP_ENABLED is TRUE, neither ON nor OFF".to_owned() - ) - ); - } - #[test] fn a_system_ggml_needs_the_runtime_only_when_its_config_searched_for_openmp() { assert_eq!( - [cache_of(&[]), cache_of(&GNU_RUNTIME)] - .map(|cache| OpenmpRuntimeSource::SystemGgml.libraries(&cache).unwrap()), - [ - Vec::new(), - vec![ - PathBuf::from("/usr/lib/gcc/libgomp.so"), - PathBuf::from("/usr/lib/libpthread.a") - ] - ] + [Vec::new(), gnu_runtime()].map(|entries| { + OpenmpRuntimeSource::SystemGgml + .libraries(&CmakeCache { entries }) + .unwrap() + }), + [Vec::new(), gnu_runtime_libraries()] ); } #[test] fn a_bundled_ggml_with_openmp_but_no_runtime_library_list_is_refused() { assert_eq!( - failure( - OpenmpRuntimeSource::BundledGgml, - &[("GGML_OPENMP_ENABLED", "ON")] - ), + bundled_ggml_failure(vec![entry("GGML_OPENMP_ENABLED", "ON")]), Some("the CMake cache has no entry OpenMP_C_LIB_NAMES".to_owned()) ); } @@ -221,14 +197,11 @@ mod tests { #[test] fn a_runtime_library_cmake_did_not_locate_is_refused() { assert_eq!( - failure( - OpenmpRuntimeSource::BundledGgml, - &[ - ("GGML_OPENMP_ENABLED", "ON"), - ("OpenMP_C_LIB_NAMES", "gomp"), - ("OpenMP_CXX_LIB_NAMES", "gomp"), - ] - ), + bundled_ggml_failure(vec![ + entry("GGML_OPENMP_ENABLED", "ON"), + entry("OpenMP_C_LIB_NAMES", "gomp"), + entry("OpenMP_CXX_LIB_NAMES", "gomp"), + ]), Some("the CMake cache has no entry OpenMP_gomp_LIBRARY".to_owned()) ); } From 8831b6c79a638f9b4a72baa6eae2a5eb6b43fa93 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 17:07:12 +0100 Subject: [PATCH 80/82] Pass test environment overrides as named variables instead of tuples --- llama-cpp-bindings-build/src/android_ndk.rs | 23 +++---- .../src/bindings_generation.rs | 10 +++- llama-cpp-bindings-build/src/build_context.rs | 60 +++++++++---------- .../src/build_environment.rs | 47 +++++++++++---- .../src/cmake_configuration.rs | 21 ++++--- llama-cpp-bindings-build/src/cpu_tuning.rs | 24 ++++---- llama-cpp-bindings-build/src/lib.rs | 3 +- .../src/link_directives.rs | 32 ++++++---- llama-cpp-bindings-build/src/target_os.rs | 14 +++-- .../src/wrapper_compilation.rs | 10 +++- 10 files changed, 146 insertions(+), 98 deletions(-) diff --git a/llama-cpp-bindings-build/src/android_ndk.rs b/llama-cpp-bindings-build/src/android_ndk.rs index 64c2a6c1b..39d2fffd1 100644 --- a/llama-cpp-bindings-build/src/android_ndk.rs +++ b/llama-cpp-bindings-build/src/android_ndk.rs @@ -193,6 +193,7 @@ pub mod tests { use super::validate_ndk_installation; use crate::build_environment::BuildEnvironment; use crate::build_environment::tests::environment_of; + use crate::build_environment::tests::variable; use crate::host_os::HostOs; pub const FIXTURE_ANDROID_NDK: &str = @@ -214,7 +215,7 @@ pub mod tests { fn ndk_at(root: &Path) -> BuildEnvironment { environment_of( BitFlags::empty(), - [("ANDROID_NDK_HOME", OsString::from(root))], + &[variable("ANDROID_NDK_HOME", OsString::from(root))], ) } @@ -238,9 +239,9 @@ pub mod tests { let detected = AndroidNdk::detect( &environment_of( BitFlags::empty(), - [ - ("ANDROID_NDK_HOME", OsString::from(FIXTURE_ANDROID_NDK)), - ("ANDROID_PLATFORM", OsString::from("android-30")), + &[ + variable("ANDROID_NDK_HOME", OsString::from(FIXTURE_ANDROID_NDK)), + variable("ANDROID_PLATFORM", OsString::from("android-30")), ], ), "aarch64-linux-android", @@ -302,9 +303,9 @@ pub mod tests { [ api_level_of(&environment_of( BitFlags::empty(), - [ - ("ANDROID_NDK_HOME", OsString::from(FIXTURE_ANDROID_NDK)), - ("ANDROID_PLATFORM", OsString::from("33")), + &[ + variable("ANDROID_NDK_HOME", OsString::from(FIXTURE_ANDROID_NDK)), + variable("ANDROID_PLATFORM", OsString::from("33")), ] )), api_level_of(&ndk_at(Path::new(FIXTURE_ANDROID_NDK))), @@ -321,7 +322,7 @@ pub mod tests { assert_eq!( [ detection_failure( - &environment_of(BitFlags::empty(), []), + &environment_of(BitFlags::empty(), &[]), "aarch64", HostOs::Linux ), @@ -330,9 +331,9 @@ pub mod tests { detection_failure( &environment_of( BitFlags::empty(), - [ - ("ANDROID_NDK_HOME", OsString::from(FIXTURE_ANDROID_NDK)), - ("ANDROID_PLATFORM", OsString::from_vec(vec![0xff])), + &[ + variable("ANDROID_NDK_HOME", OsString::from(FIXTURE_ANDROID_NDK)), + variable("ANDROID_PLATFORM", OsString::from_vec(vec![0xff])), ] ), "aarch64", diff --git a/llama-cpp-bindings-build/src/bindings_generation.rs b/llama-cpp-bindings-build/src/bindings_generation.rs index 4608a9d2d..8c8d7565d 100644 --- a/llama-cpp-bindings-build/src/bindings_generation.rs +++ b/llama-cpp-bindings-build/src/bindings_generation.rs @@ -216,14 +216,18 @@ mod tests { use super::BindingsGeneration; use crate::android_ndk::tests::FIXTURE_ANDROID_NDK; - use crate::build_context::tests::ANDROID_OVERRIDES; + use crate::build_context::tests::android_overrides; use crate::build_context::tests::cargo_environment; use crate::build_context::tests::context_of; + use crate::build_environment::tests::EnvironmentVariable; use crate::build_feature::BuildFeature; const FIXTURE_BINDINGS: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/fixtures/bindings"); - fn planned(features: BitFlags, overrides: &[(&str, &str)]) -> BindingsGeneration { + fn planned( + features: BitFlags, + overrides: &[EnvironmentVariable], + ) -> BindingsGeneration { let environment = cargo_environment(features, overrides); BindingsGeneration::of(&context_of(&environment), &environment) @@ -277,7 +281,7 @@ mod tests { let toolchain = format!("{FIXTURE_ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64"); assert_eq!( - planned(BitFlags::empty(), &ANDROID_OVERRIDES).clang_args, + planned(BitFlags::empty(), &android_overrides()).clang_args, [ llama_cpp_includes(), vec![ diff --git a/llama-cpp-bindings-build/src/build_context.rs b/llama-cpp-bindings-build/src/build_context.rs index 91b982e0d..b8261434b 100644 --- a/llama-cpp-bindings-build/src/build_context.rs +++ b/llama-cpp-bindings-build/src/build_context.rs @@ -86,7 +86,6 @@ impl BuildContext { #[cfg(test)] pub mod tests { use std::ffi::OsStr; - use std::ffi::OsString; use std::path::PathBuf; use enumflags2::BitFlags; @@ -94,48 +93,43 @@ pub mod tests { use super::BuildContext; use crate::android_ndk::tests::FIXTURE_ANDROID_NDK; use crate::build_environment::BuildEnvironment; + use crate::build_environment::tests::EnvironmentVariable; use crate::build_environment::tests::environment_of; + use crate::build_environment::tests::variable; use crate::build_environment::tests::with_non_unicode; use crate::build_feature::BuildFeature; use crate::cargo_directive::CargoDirective; use crate::host_os::HostOs; use crate::target_os::TargetOs; - const CARGO_VARIABLES: [(&str, &str); 8] = [ - ("CARGO_CFG_TARGET_ARCH", "x86_64"), - ("CARGO_CFG_TARGET_OS", "linux"), - ("CARGO_MANIFEST_DIR", "/sys"), - ("DEBUG", "false"), - ("HOST", "x86_64-unknown-linux-gnu"), - ("OPT_LEVEL", "3"), - ("OUT_DIR", "/out"), - ("TARGET", "x86_64-unknown-linux-gnu"), - ]; - - pub const ANDROID_OVERRIDES: [(&str, &str); 4] = [ - ("ANDROID_NDK_HOME", FIXTURE_ANDROID_NDK), - ("CARGO_CFG_TARGET_ARCH", "aarch64"), - ("CARGO_CFG_TARGET_OS", "android"), - ("TARGET", "aarch64-linux-android"), - ]; + fn cargo_variables() -> [EnvironmentVariable; 8] { + [ + variable("CARGO_CFG_TARGET_ARCH", "x86_64"), + variable("CARGO_CFG_TARGET_OS", "linux"), + variable("CARGO_MANIFEST_DIR", "/sys"), + variable("DEBUG", "false"), + variable("HOST", "x86_64-unknown-linux-gnu"), + variable("OPT_LEVEL", "3"), + variable("OUT_DIR", "/out"), + variable("TARGET", "x86_64-unknown-linux-gnu"), + ] + } + + pub fn android_overrides() -> [EnvironmentVariable; 4] { + [ + variable("ANDROID_NDK_HOME", FIXTURE_ANDROID_NDK), + variable("CARGO_CFG_TARGET_ARCH", "aarch64"), + variable("CARGO_CFG_TARGET_OS", "android"), + variable("TARGET", "aarch64-linux-android"), + ] + } /// The variables cargo sets for a build script building for x86-64 Linux, with `overrides`. pub fn cargo_environment( features: BitFlags, - overrides: &[(&str, &str)], + overrides: &[EnvironmentVariable], ) -> BuildEnvironment { - let mut environment = environment_of( - features, - CARGO_VARIABLES.map(|(name, value)| (name, OsString::from(value))), - ); - - environment.variables.extend( - overrides - .iter() - .map(|(name, value)| (OsString::from(name), OsString::from(value))), - ); - - environment + environment_of(features, &[&cargo_variables()[..], overrides].concat()) } pub fn context_of(environment: &BuildEnvironment) -> BuildContext { @@ -147,7 +141,7 @@ pub mod tests { assert_eq!( context_of(&cargo_environment( BitFlags::from(BuildFeature::DynamicLink), - &[("BUILD_DEBUG", "")] + &[variable("BUILD_DEBUG", "")] )), BuildContext { out_dir: PathBuf::from("/out"), @@ -175,7 +169,7 @@ pub mod tests { .err() .map(|error| error.to_string()) }; - let required = CARGO_VARIABLES.map(|(name, _)| name); + let required = cargo_variables().map(|EnvironmentVariable { name, .. }| name); assert_eq!( required.map(without), diff --git a/llama-cpp-bindings-build/src/build_environment.rs b/llama-cpp-bindings-build/src/build_environment.rs index 998bfda58..472c8be46 100644 --- a/llama-cpp-bindings-build/src/build_environment.rs +++ b/llama-cpp-bindings-build/src/build_environment.rs @@ -83,19 +83,41 @@ pub mod tests { use crate::build_feature::BuildFeature; use crate::enabled_build_features::enabled_build_features; + #[derive(Clone, Debug)] + pub struct EnvironmentVariable { + pub name: &'static str, + pub value: OsString, + } + + #[must_use] + pub fn variable>( + name: &'static str, + value: TValue, + ) -> EnvironmentVariable { + EnvironmentVariable { + name, + value: value.into(), + } + } + #[must_use] - pub fn environment_of( + pub fn environment_of( features: BitFlags, - variables: [(&str, OsString); VARIABLES], + variables: &[EnvironmentVariable], ) -> BuildEnvironment { - BuildEnvironment { - variables: variables - .into_iter() - .map(|(name, value)| (OsString::from(name), value)) - .collect::>(), + let mut environment = BuildEnvironment { + variables: BTreeMap::new(), features, cuda_library_dirs: Vec::new(), + }; + + for EnvironmentVariable { name, value } in variables { + environment + .variables + .insert(OsString::from(name), value.clone()); } + + environment } #[must_use] @@ -123,7 +145,10 @@ pub mod tests { fn a_unicode_variable_is_read() { let environment = environment_of( BitFlags::empty(), - [("TARGET", OsString::from("x86_64-unknown-linux-gnu"))], + &[variable( + "TARGET", + OsString::from("x86_64-unknown-linux-gnu"), + )], ); assert_eq!( @@ -137,7 +162,7 @@ pub mod tests { #[test] fn a_missing_variable_fails_when_required_and_is_none_when_optional() { - let environment = environment_of(BitFlags::empty(), []); + let environment = environment_of(BitFlags::empty(), &[]); assert_eq!(environment.optional("HOST").ok(), Some(None)); assert_eq!( @@ -154,7 +179,7 @@ pub mod tests { #[test] fn a_variable_that_is_not_unicode_is_refused_even_when_optional() { - let environment = with_non_unicode(environment_of(BitFlags::empty(), []), "TARGET"); + let environment = with_non_unicode(environment_of(BitFlags::empty(), &[]), "TARGET"); assert_eq!( environment @@ -171,7 +196,7 @@ pub mod tests { #[test] fn features_are_reported_by_membership() { - let environment = environment_of(BitFlags::from(BuildFeature::Cuda), []); + let environment = environment_of(BitFlags::from(BuildFeature::Cuda), &[]); assert_eq!( [ diff --git a/llama-cpp-bindings-build/src/cmake_configuration.rs b/llama-cpp-bindings-build/src/cmake_configuration.rs index 2d1eb7ae4..248809f8a 100644 --- a/llama-cpp-bindings-build/src/cmake_configuration.rs +++ b/llama-cpp-bindings-build/src/cmake_configuration.rs @@ -209,9 +209,11 @@ mod tests { use super::CmakeConfiguration; use crate::android_ndk::tests::FIXTURE_ANDROID_NDK; - use crate::build_context::tests::ANDROID_OVERRIDES; + use crate::build_context::tests::android_overrides; use crate::build_context::tests::cargo_environment; use crate::build_context::tests::context_of; + use crate::build_environment::tests::EnvironmentVariable; + use crate::build_environment::tests::variable; use crate::build_environment::tests::with_non_unicode; use crate::build_feature::BuildFeature; use crate::cargo_directive::CargoDirective; @@ -222,7 +224,7 @@ mod tests { fn configured( features: BitFlags, - overrides: &[(&str, &str)], + overrides: &[EnvironmentVariable], ) -> CmakeConfiguration { let environment = cargo_environment(features, overrides); @@ -231,7 +233,7 @@ mod tests { fn after_the_base_switches( features: BitFlags, - overrides: &[(&str, &str)], + overrides: &[EnvironmentVariable], ) -> CmakeConfiguration { let environment = cargo_environment(features, overrides); let context = context_of(&environment); @@ -298,7 +300,7 @@ mod tests { assert_eq!( after_the_base_switches( BuildFeature::Metal | BuildFeature::Vulkan, - &[("CARGO_CFG_TARGET_OS", "macos")] + &[variable("CARGO_CFG_TARGET_OS", "macos")] ), CmakeConfiguration { defines: vec![ @@ -317,7 +319,7 @@ mod tests { #[test] fn android_targets_build_with_the_ndk_toolchain_their_stl_and_no_openmp() { let android_with = |features: BitFlags| { - after_the_base_switches(features | BuildFeature::Openmp, &ANDROID_OVERRIDES) + after_the_base_switches(features | BuildFeature::Openmp, &android_overrides()) }; let android_defines = |stl: Vec| { [ @@ -366,7 +368,7 @@ mod tests { #[test] fn gpu_backends_turn_on_their_ggml_switches_and_take_their_sdk_and_architectures_when_set() { - let gpu_configuration = |overrides: &[(&str, &str)]| { + let gpu_configuration = |overrides: &[EnvironmentVariable]| { after_the_base_switches( BuildFeature::Vulkan | BuildFeature::Cuda @@ -405,7 +407,10 @@ mod tests { assert_eq!( [ - gpu_configuration(&[("VULKAN_SDK", "/vulkan"), ("CUDAARCHS", "89")]), + gpu_configuration(&[ + variable("VULKAN_SDK", "/vulkan"), + variable("CUDAARCHS", "89") + ]), gpu_configuration(&[]), ], [ @@ -441,7 +446,7 @@ mod tests { assert_eq!( after_the_base_switches( BitFlags::from(BuildFeature::SystemGgml), - &[("GGML_DIR", "/ggml/lib/cmake/ggml")] + &[variable("GGML_DIR", "/ggml/lib/cmake/ggml")] ), CmakeConfiguration { defines: vec![ diff --git a/llama-cpp-bindings-build/src/cpu_tuning.rs b/llama-cpp-bindings-build/src/cpu_tuning.rs index 86cff30d4..b0ab197d5 100644 --- a/llama-cpp-bindings-build/src/cpu_tuning.rs +++ b/llama-cpp-bindings-build/src/cpu_tuning.rs @@ -209,12 +209,14 @@ mod cpu_tuning_tests { use super::CpuTuning; use crate::build_context::tests::cargo_environment; use crate::build_context::tests::context_of; + use crate::build_environment::tests::EnvironmentVariable; + use crate::build_environment::tests::variable; use crate::build_environment::tests::with_non_unicode; use crate::build_feature::BuildFeature; use crate::cargo_directive::CargoDirective; use crate::cmake_define::CmakeDefine; - fn tuned(features: BitFlags, overrides: &[(&str, &str)]) -> CpuTuning { + fn tuned(features: BitFlags, overrides: &[EnvironmentVariable]) -> CpuTuning { let environment = cargo_environment(features, overrides); CpuTuning::of(&context_of(&environment), &environment).unwrap() @@ -248,8 +250,8 @@ mod cpu_tuning_tests { tuned( BitFlags::empty(), &[ - ("CARGO_ENCODED_RUSTFLAGS", "-Ctarget-cpu=native"), - ("CARGO_CFG_TARGET_FEATURE", "avx2"), + variable("CARGO_ENCODED_RUSTFLAGS", "-Ctarget-cpu=native"), + variable("CARGO_CFG_TARGET_FEATURE", "avx2"), ] ), CpuTuning { @@ -266,7 +268,7 @@ mod cpu_tuning_tests { assert_eq!( tuned( BitFlags::empty(), - &[("CARGO_CFG_TARGET_FEATURE", "fxsr,sse,sse2")] + &[variable("CARGO_CFG_TARGET_FEATURE", "fxsr,sse,sse2")] ), CpuTuning { defines: [ @@ -287,8 +289,8 @@ mod cpu_tuning_tests { tuned( BitFlags::empty(), &[ - ("CARGO_ENCODED_RUSTFLAGS", "-Ctarget-cpu=znver4"), - ( + variable("CARGO_ENCODED_RUSTFLAGS", "-Ctarget-cpu=znver4"), + variable( "CARGO_CFG_TARGET_FEATURE", "avx,avx2,avx512f,avx512cd,avx512vl,avx512dq,bmi2,f16c,fma,sse4.2" ), @@ -316,8 +318,8 @@ mod cpu_tuning_tests { tuned( BitFlags::empty(), &[ - ("CARGO_CFG_TARGET_ARCH", "aarch64"), - ("TARGET", "aarch64-unknown-linux-gnu"), + variable("CARGO_CFG_TARGET_ARCH", "aarch64"), + variable("TARGET", "aarch64-unknown-linux-gnu"), ] ), CpuTuning { @@ -335,9 +337,9 @@ mod cpu_tuning_tests { tuned( BuildFeature::DynamicBackends | BuildFeature::DynamicLink, &[ - ("CARGO_CFG_TARGET_ARCH", "aarch64"), - ("TARGET", "aarch64-unknown-linux-gnu"), - ("CARGO_ENCODED_RUSTFLAGS", "-Ctarget-cpu=native"), + variable("CARGO_CFG_TARGET_ARCH", "aarch64"), + variable("TARGET", "aarch64-unknown-linux-gnu"), + variable("CARGO_ENCODED_RUSTFLAGS", "-Ctarget-cpu=native"), ] ), CpuTuning { diff --git a/llama-cpp-bindings-build/src/lib.rs b/llama-cpp-bindings-build/src/lib.rs index 8f127e172..a23bb974d 100644 --- a/llama-cpp-bindings-build/src/lib.rs +++ b/llama-cpp-bindings-build/src/lib.rs @@ -68,6 +68,7 @@ mod tests { use super::build; use crate::build_context::tests::cargo_environment; use crate::build_environment::BuildEnvironment; + use crate::build_environment::tests::variable; use crate::build_feature::BuildFeature; const FIXTURE_CMAKE_PROJECT: &str = @@ -100,7 +101,7 @@ mod tests { failure(&cargo_environment(BitFlags::from(BuildFeature::Rocm), &[])), failure(&cargo_environment( BitFlags::empty(), - &[("CARGO_MANIFEST_DIR", FIXTURE_CMAKE_PROJECT)] + &[variable("CARGO_MANIFEST_DIR", FIXTURE_CMAKE_PROJECT)] )), ], [ diff --git a/llama-cpp-bindings-build/src/link_directives.rs b/llama-cpp-bindings-build/src/link_directives.rs index 47be2ee48..5b08ee3db 100644 --- a/llama-cpp-bindings-build/src/link_directives.rs +++ b/llama-cpp-bindings-build/src/link_directives.rs @@ -266,10 +266,12 @@ mod tests { use enumflags2::BitFlags; use super::link_directives; - use crate::build_context::tests::ANDROID_OVERRIDES; + use crate::build_context::tests::android_overrides; use crate::build_context::tests::cargo_environment; use crate::build_context::tests::context_of; use crate::build_environment::BuildEnvironment; + use crate::build_environment::tests::EnvironmentVariable; + use crate::build_environment::tests::variable; use crate::build_feature::BuildFeature; use crate::cargo_directive::CargoDirective; use crate::link_kind::LinkKind; @@ -303,7 +305,7 @@ mod tests { fn platform_libraries( features: BitFlags, - overrides: &[(&str, &str)], + overrides: &[EnvironmentVariable], ) -> Vec { let libraries = libraries(&linked(&cargo_environment(features, overrides))); @@ -422,7 +424,7 @@ mod tests { | BuildFeature::Metal | BuildFeature::Vulkan | BuildFeature::Rocm, - &[("ROCM_PATH", rocm_root.to_str().unwrap())], + &[variable("ROCM_PATH", rocm_root.to_str().unwrap())], ) }; let directives = linked(&environment); @@ -506,7 +508,7 @@ mod tests { let system_ggml_linked = |features: BitFlags| { linked(&cargo_environment( features | BuildFeature::SystemGgml | BuildFeature::Cuda, - &[("GGML_LIBRARY_DIR", FIXTURES)], + &[variable("GGML_LIBRARY_DIR", FIXTURES)], )) }; let linked_with_ggml_as = |ggml_kind: LinkKind| { @@ -564,18 +566,21 @@ mod tests { [ platform_libraries( BitFlags::from(BuildFeature::Metal), - &[("CARGO_CFG_TARGET_OS", "macos")] + &[variable("CARGO_CFG_TARGET_OS", "macos")] + ), + platform_libraries( + BitFlags::empty(), + &[variable("CARGO_CFG_TARGET_OS", "macos")] ), - platform_libraries(BitFlags::empty(), &[("CARGO_CFG_TARGET_OS", "macos")]), platform_libraries( BitFlags::from(BuildFeature::StaticStdcxx), - &ANDROID_OVERRIDES + &android_overrides() ), platform_libraries( BitFlags::from(BuildFeature::SharedStdcxx), - &ANDROID_OVERRIDES + &android_overrides() ), - platform_libraries(BitFlags::empty(), &ANDROID_OVERRIDES), + platform_libraries(BitFlags::empty(), &android_overrides()), ], [ vec![ @@ -605,7 +610,7 @@ mod tests { "llama-cpp-bindings-build-missing-{}", process::id() )); - let refusal = |features: BitFlags, overrides: &[(&str, &str)]| { + let refusal = |features: BitFlags, overrides: &[EnvironmentVariable]| { let environment = cargo_environment(features, overrides); link_directives(&context_of(&environment), &environment) @@ -618,12 +623,15 @@ mod tests { refusal(BitFlags::from(BuildFeature::SystemGgml), &[]), refusal( BitFlags::from(BuildFeature::SystemGgml), - &[("GGML_LIBRARY_DIR", missing_directory.to_str().unwrap())] + &[variable( + "GGML_LIBRARY_DIR", + missing_directory.to_str().unwrap() + )] ), refusal(BitFlags::from(BuildFeature::Rocm), &[]), refusal( BitFlags::from(BuildFeature::Rocm), - &[("ROCM_PATH", env!("CARGO_MANIFEST_DIR"))] + &[variable("ROCM_PATH", env!("CARGO_MANIFEST_DIR"))] ), ], [ diff --git a/llama-cpp-bindings-build/src/target_os.rs b/llama-cpp-bindings-build/src/target_os.rs index e3b46c781..c346f9b37 100644 --- a/llama-cpp-bindings-build/src/target_os.rs +++ b/llama-cpp-bindings-build/src/target_os.rs @@ -53,14 +53,15 @@ mod tests { use crate::android_ndk::AndroidNdk; use crate::android_ndk::tests::FIXTURE_ANDROID_NDK; use crate::build_environment::tests::environment_of; + use crate::build_environment::tests::variable; use crate::host_os::HostOs; fn detected(cargo_cfg_target_os: &str) -> Result { let environment = environment_of( BitFlags::empty(), - [ - ("CARGO_CFG_TARGET_OS", OsString::from(cargo_cfg_target_os)), - ("ANDROID_NDK_HOME", OsString::from(FIXTURE_ANDROID_NDK)), + &[ + variable("CARGO_CFG_TARGET_OS", OsString::from(cargo_cfg_target_os)), + variable("ANDROID_NDK_HOME", OsString::from(FIXTURE_ANDROID_NDK)), ], ); @@ -85,7 +86,10 @@ mod tests { fn android_is_not_mistaken_for_linux_and_carries_its_ndk() { let environment = environment_of( BitFlags::empty(), - [("ANDROID_NDK_HOME", OsString::from(FIXTURE_ANDROID_NDK))], + &[variable( + "ANDROID_NDK_HOME", + OsString::from(FIXTURE_ANDROID_NDK), + )], ); assert_eq!( @@ -119,7 +123,7 @@ mod tests { TargetOs::detect( &environment_of( BitFlags::empty(), - [("CARGO_CFG_TARGET_OS", "android".into())] + &[variable("CARGO_CFG_TARGET_OS", "android")] ), "aarch64-linux-android", "aarch64", diff --git a/llama-cpp-bindings-build/src/wrapper_compilation.rs b/llama-cpp-bindings-build/src/wrapper_compilation.rs index 948388908..657474f99 100644 --- a/llama-cpp-bindings-build/src/wrapper_compilation.rs +++ b/llama-cpp-bindings-build/src/wrapper_compilation.rs @@ -106,12 +106,16 @@ mod tests { use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; use super::WrapperCompilation; - use crate::build_context::tests::ANDROID_OVERRIDES; + use crate::build_context::tests::android_overrides; use crate::build_context::tests::cargo_environment; use crate::build_context::tests::context_of; + use crate::build_environment::tests::EnvironmentVariable; use crate::build_feature::BuildFeature; - fn planned(features: BitFlags, overrides: &[(&str, &str)]) -> WrapperCompilation { + fn planned( + features: BitFlags, + overrides: &[EnvironmentVariable], + ) -> WrapperCompilation { let environment = cargo_environment(features, overrides); WrapperCompilation::of(&context_of(&environment), &environment) @@ -157,7 +161,7 @@ mod tests { assert_eq!( planned( BitFlags::from(BuildFeature::StaticStdcxx), - &ANDROID_OVERRIDES + &android_overrides() ), WrapperCompilation { cpp_stdlib_linked_separately: true, From 8bd8b55f3cb8ccdaba02b4b4de85ab958eac89de Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 18:28:16 +0100 Subject: [PATCH 81/82] Declare intentional test failures with their exact typed failure and reason, provisioning models in their own crate --- Cargo.lock | 12 +- Cargo.toml | 2 + Makefile | 4 +- llama-cpp-bindings-tests/Cargo.toml | 1 + .../tests/lora_adapters.rs | 2 +- .../tests/parameter_fitting.rs | 2 +- .../src/attribute_accumulator.rs | 2 + llama-cpp-test-harness-macros/src/expand.rs | 142 ++++++++++++-- llama-cpp-test-harness-macros/src/lib.rs | 4 +- .../src/parsed_args.rs | 27 ++- .../src/parsed_expected_failure.rs | 145 ++++++++++++++ .../src/parsed_expected_outcome.rs | 7 + llama-cpp-test-harness/Cargo.toml | 2 +- llama-cpp-test-harness/src/download_model.rs | 31 --- llama-cpp-test-harness/src/execution_phase.rs | 9 +- llama-cpp-test-harness/src/execution_plan.rs | 11 +- .../src/expected_failure.rs | 37 ++++ .../src/expected_outcome.rs | 184 ++++++++++++++++++ llama-cpp-test-harness/src/gguf_source.rs | 72 ------- .../src/harness_arguments_error.rs | 9 - .../src/harness_run_error.rs | 8 +- llama-cpp-test-harness/src/lib.rs | 13 +- .../src/llama_test_registration.rs | 5 +- .../src/parse_harness_arguments.rs | 14 +- .../src/phase_model_files.rs | 101 ---------- llama-cpp-test-harness/src/run.rs | 2 +- .../src/run_to_conclusions.rs | 7 +- llama-cpp-test-harness/src/scheduled_phase.rs | 139 +++++++++---- .../src/test_backend_gate.rs | 4 - llama-cpp-test-harness/src/trial_outcome.rs | 27 +++ .../tests/phase_conclusions.rs | 49 ++--- llama-cpp-test-model-provisioning/Cargo.toml | 22 +++ .../src/backend_init_gate.rs | 5 + .../src/download_model.rs | 15 ++ .../src/gguf_source.rs | 39 ++++ llama-cpp-test-model-provisioning/src/lib.rs | 15 ++ .../src/load_key.rs | 67 +++++-- .../src/model_load_params.rs | 0 .../src/phase_model_files.rs | 136 +++++++++++++ .../src/phase_setup.rs | 14 +- .../src/phase_setup_error.rs | 51 +++++ .../src/phase_setup_failure_kind.rs | 27 +++ .../src/phase_state.rs | 0 43 files changed, 1104 insertions(+), 361 deletions(-) create mode 100644 llama-cpp-test-harness-macros/src/parsed_expected_failure.rs create mode 100644 llama-cpp-test-harness-macros/src/parsed_expected_outcome.rs delete mode 100644 llama-cpp-test-harness/src/download_model.rs create mode 100644 llama-cpp-test-harness/src/expected_failure.rs create mode 100644 llama-cpp-test-harness/src/expected_outcome.rs delete mode 100644 llama-cpp-test-harness/src/gguf_source.rs delete mode 100644 llama-cpp-test-harness/src/harness_arguments_error.rs delete mode 100644 llama-cpp-test-harness/src/phase_model_files.rs delete mode 100644 llama-cpp-test-harness/src/test_backend_gate.rs create mode 100644 llama-cpp-test-harness/src/trial_outcome.rs create mode 100644 llama-cpp-test-model-provisioning/Cargo.toml create mode 100644 llama-cpp-test-model-provisioning/src/backend_init_gate.rs create mode 100644 llama-cpp-test-model-provisioning/src/download_model.rs create mode 100644 llama-cpp-test-model-provisioning/src/gguf_source.rs create mode 100644 llama-cpp-test-model-provisioning/src/lib.rs rename {llama-cpp-test-harness => llama-cpp-test-model-provisioning}/src/load_key.rs (63%) rename {llama-cpp-test-harness => llama-cpp-test-model-provisioning}/src/model_load_params.rs (100%) create mode 100644 llama-cpp-test-model-provisioning/src/phase_model_files.rs rename {llama-cpp-test-harness => llama-cpp-test-model-provisioning}/src/phase_setup.rs (91%) create mode 100644 llama-cpp-test-model-provisioning/src/phase_setup_error.rs create mode 100644 llama-cpp-test-model-provisioning/src/phase_setup_failure_kind.rs rename {llama-cpp-test-harness => llama-cpp-test-model-provisioning}/src/phase_state.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 406c19b58..9186550d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1222,6 +1222,7 @@ dependencies = [ "llama-cpp-gbnf", "llama-cpp-test-harness", "llama-cpp-test-harness-macros", + "llama-cpp-test-model-provisioning", "serde_json", "toktrie", ] @@ -1276,11 +1277,11 @@ name = "llama-cpp-test-harness" version = "0.26.0" dependencies = [ "anyhow", - "hf-hub", "inventory", "libtest-mimic", "llama-cpp-bindings", "llama-cpp-test-harness-macros", + "llama-cpp-test-model-provisioning", "thiserror", ] @@ -1293,6 +1294,15 @@ dependencies = [ "syn", ] +[[package]] +name = "llama-cpp-test-model-provisioning" +version = "0.26.0" +dependencies = [ + "hf-hub", + "llama-cpp-bindings", + "thiserror", +] + [[package]] name = "llama-cpp-wrapper-error-fixture" version = "0.26.0" diff --git a/Cargo.toml b/Cargo.toml index 7da283e83..01301246b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ members = [ "llama-cpp-log-decoder", "llama-cpp-test-harness", "llama-cpp-test-harness-macros", + "llama-cpp-test-model-provisioning", "llama-cpp-wrapper-error-fixture", "llama-cpp-wrapper-sources", ] @@ -44,6 +45,7 @@ llama-cpp-gbnf = { path = "llama-cpp-gbnf", version = "=0.26.0" } llama-cpp-log-decoder = { path = "llama-cpp-log-decoder", version = "=0.26.0" } llama-cpp-test-harness = { path = "llama-cpp-test-harness", version = "=0.26.0" } llama-cpp-test-harness-macros = { path = "llama-cpp-test-harness-macros", version = "=0.26.0" } +llama-cpp-test-model-provisioning = { path = "llama-cpp-test-model-provisioning", version = "=0.26.0" } llama-cpp-fault-injection = { path = "llama-cpp-fault-injection" } llama-cpp-wrapper-error-fixture = { path = "llama-cpp-wrapper-error-fixture" } llama-cpp-wrapper-sources = { path = "llama-cpp-wrapper-sources", version = "=0.26.0" } diff --git a/Makefile b/Makefile index e8b9dc16e..a20ec822f 100644 --- a/Makefile +++ b/Makefile @@ -83,6 +83,7 @@ coverage: node_modules --gated llama-cpp-log-decoder=100 \ --gated llama-cpp-test-harness=100 \ --gated llama-cpp-test-harness-macros=100 \ + --gated llama-cpp-test-model-provisioning=100 \ --gated llama-cpp-wrapper-sources=100 .PHONY: coverage-clean @@ -122,7 +123,8 @@ test: test.llms .PHONY: test.harness test.harness: clippy - cargo test -p llama-cpp-test-harness-macros -p llama-cpp-test-harness $(DEVICE_FEATURE) + cargo test -p llama-cpp-test-harness-macros -p llama-cpp-test-harness \ + -p llama-cpp-test-model-provisioning $(DEVICE_FEATURE) .PHONY: test.llms test.llms: clippy test.harness test.unit diff --git a/llama-cpp-bindings-tests/Cargo.toml b/llama-cpp-bindings-tests/Cargo.toml index 53c5d036f..e873ffe65 100644 --- a/llama-cpp-bindings-tests/Cargo.toml +++ b/llama-cpp-bindings-tests/Cargo.toml @@ -19,6 +19,7 @@ llama-cpp-ffi-status = { workspace = true } llama-cpp-gbnf = { workspace = true } llama-cpp-test-harness = { workspace = true } llama-cpp-test-harness-macros = { workspace = true } +llama-cpp-test-model-provisioning = { workspace = true } serde_json = { workspace = true } toktrie = { workspace = true } diff --git a/llama-cpp-bindings-tests/tests/lora_adapters.rs b/llama-cpp-bindings-tests/tests/lora_adapters.rs index 7ccf4e72c..cebe2f2f9 100644 --- a/llama-cpp-bindings-tests/tests/lora_adapters.rs +++ b/llama-cpp-bindings-tests/tests/lora_adapters.rs @@ -13,9 +13,9 @@ use llama_cpp_bindings::error::llama_lora_adapter_init_error::LlamaLoraAdapterIn use llama_cpp_bindings::error::llama_lora_adapters_error::LlamaLoraAdaptersError; use llama_cpp_bindings::model::lora_adapter_scale::LoraAdapterScale; use llama_cpp_fault_injection::armed_fault::ArmedFault; -use llama_cpp_test_harness::download_model::download_model; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; +use llama_cpp_test_model_provisioning::download_model::download_model; #[llama_test( model_source = HuggingFace( diff --git a/llama-cpp-bindings-tests/tests/parameter_fitting.rs b/llama-cpp-bindings-tests/tests/parameter_fitting.rs index db79904a9..deac94da7 100644 --- a/llama-cpp-bindings-tests/tests/parameter_fitting.rs +++ b/llama-cpp-bindings-tests/tests/parameter_fitting.rs @@ -10,9 +10,9 @@ use llama_cpp_bindings::model::params::LlamaModelParams; use llama_cpp_bindings::model::params::fit_extra_model::FitExtraModel; use llama_cpp_bindings::mtmd::mtmd_context_params::MtmdContextParams; use llama_cpp_bindings::mtmd::mtmd_memory_usage::MtmdMemoryUsage; -use llama_cpp_test_harness::gguf_source::GgufSource; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; +use llama_cpp_test_model_provisioning::gguf_source::GgufSource; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), diff --git a/llama-cpp-test-harness-macros/src/attribute_accumulator.rs b/llama-cpp-test-harness-macros/src/attribute_accumulator.rs index c5160ed2c..b7e677c54 100644 --- a/llama-cpp-test-harness-macros/src/attribute_accumulator.rs +++ b/llama-cpp-test-harness-macros/src/attribute_accumulator.rs @@ -1,3 +1,4 @@ +use crate::parsed_expected_failure::ParsedExpectedFailure; use crate::parsed_load_mode::ParsedLoadMode; use crate::parsed_source::ParsedSource; @@ -13,4 +14,5 @@ pub struct AttributeAccumulator { pub embeddings: Option, pub n_seq_max: Option, pub n_threads_batch: Option, + pub expected_failure: Option, } diff --git a/llama-cpp-test-harness-macros/src/expand.rs b/llama-cpp-test-harness-macros/src/expand.rs index 609c5303c..e3b37e25f 100644 --- a/llama-cpp-test-harness-macros/src/expand.rs +++ b/llama-cpp-test-harness-macros/src/expand.rs @@ -10,6 +10,8 @@ use syn::parse::ParseStream; use syn::parse2; use crate::parsed_args::ParsedArgs; +use crate::parsed_expected_failure::ParsedExpectedFailure; +use crate::parsed_expected_outcome::ParsedExpectedOutcome; use crate::parsed_source::ParsedSource; struct StackedItems { @@ -73,13 +75,13 @@ fn split_fn_and_pass_through(items: Vec) -> syn::Result<(ItemFn, Vec fn build_model_source_literal(source: &ParsedSource) -> TokenStream { match source { ParsedSource::HuggingFace { repo, file } => quote! { - ::llama_cpp_test_harness::gguf_source::GgufSource::HuggingFace { + ::llama_cpp_test_model_provisioning::gguf_source::GgufSource::HuggingFace { repo: #repo, file: #file, } }, ParsedSource::LocalPath(path) => quote! { - ::llama_cpp_test_harness::gguf_source::GgufSource::LocalPath(#path) + ::llama_cpp_test_model_provisioning::gguf_source::GgufSource::LocalPath(#path) }, } } @@ -88,13 +90,13 @@ fn build_mmproj_source_literal(source: Option<&ParsedSource>) -> TokenStream { match source { None => quote! { ::core::option::Option::None }, Some(ParsedSource::HuggingFace { repo, file }) => quote! { - ::core::option::Option::Some(::llama_cpp_test_harness::gguf_source::GgufSource::HuggingFace { + ::core::option::Option::Some(::llama_cpp_test_model_provisioning::gguf_source::GgufSource::HuggingFace { repo: #repo, file: #file, }) }, Some(ParsedSource::LocalPath(path)) => quote! { - ::core::option::Option::Some(::llama_cpp_test_harness::gguf_source::GgufSource::LocalPath(#path)) + ::core::option::Option::Some(::llama_cpp_test_model_provisioning::gguf_source::GgufSource::LocalPath(#path)) }, } } @@ -112,6 +114,49 @@ fn build_test_fn_literal(signature: &Signature) -> TokenStream { } } +fn validate_expected_outcome( + expected_outcome: &ParsedExpectedOutcome, + signature: &Signature, +) -> syn::Result<()> { + if let ( + ParsedExpectedOutcome::Failure(ParsedExpectedFailure::Test { .. }), + ReturnType::Default, + ) = (expected_outcome, &signature.output) + { + Err(syn::Error::new_spanned( + signature, + "`expected_failure = TestFailure(...)` needs a test that returns a `Result`, because \ + only an error it returns counts as its failure", + )) + } else { + Ok(()) + } +} + +fn build_expected_outcome_literal(expected_outcome: &ParsedExpectedOutcome) -> TokenStream { + match expected_outcome { + ParsedExpectedOutcome::Success => { + quote! { ::llama_cpp_test_harness::expected_outcome::ExpectedOutcome::Success } + } + ParsedExpectedOutcome::Failure(ParsedExpectedFailure::PhaseSetup { kind, reason }) => { + quote! { + ::llama_cpp_test_harness::expected_outcome::ExpectedOutcome::Failure { + failure: ::llama_cpp_test_harness::expected_failure::ExpectedFailure::PhaseSetup( + ::llama_cpp_test_model_provisioning::phase_setup_failure_kind::PhaseSetupFailureKind::#kind + ), + reason: #reason, + } + } + } + ParsedExpectedOutcome::Failure(ParsedExpectedFailure::Test { reason }) => quote! { + ::llama_cpp_test_harness::expected_outcome::ExpectedOutcome::Failure { + failure: ::llama_cpp_test_harness::expected_failure::ExpectedFailure::Test, + reason: #reason, + } + }, + } +} + fn build_registration(args: &ParsedArgs, signature: &Signature) -> TokenStream { let fn_name = &signature.ident; let trial_name = format!( @@ -128,6 +173,7 @@ fn build_registration(args: &ParsedArgs, signature: &Signature) -> TokenStream { let embeddings_flag = args.context_params.embeddings; let sequence_max = args.context_params.n_seq_max; let test_fn = build_test_fn_literal(signature); + let expected_outcome = build_expected_outcome_literal(&args.expected_outcome); let threads_batch = args.context_params.n_threads_batch.map_or_else( || quote! { ::core::option::Option::None }, |value| quote! { ::core::option::Option::Some(#value) }, @@ -137,10 +183,10 @@ fn build_registration(args: &ParsedArgs, signature: &Signature) -> TokenStream { ::inventory::submit! { ::llama_cpp_test_harness::llama_test_registration::LlamaTestRegistration { name: #trial_name, - key: ::llama_cpp_test_harness::load_key::LoadKey { + key: ::llama_cpp_test_model_provisioning::load_key::LoadKey { model_source: #model_source_literal, mmproj_source: #mmproj_source_literal, - model_load_params: ::llama_cpp_test_harness::model_load_params::ModelLoadParams { + model_load_params: ::llama_cpp_test_model_provisioning::model_load_params::ModelLoadParams { n_gpu_layers: #gpu_layers, load_mode: #load_mode, }, @@ -154,6 +200,7 @@ fn build_registration(args: &ParsedArgs, signature: &Signature) -> TokenStream { embeddings: #embeddings_flag, }, func: #test_fn, + expected_outcome: #expected_outcome, } } } @@ -164,6 +211,7 @@ pub fn expand(attribute: TokenStream, item: TokenStream) -> syn::Result syn::Result TokenStream { @@ -227,13 +280,13 @@ mod tests { ::inventory::submit! { ::llama_cpp_test_harness::llama_test_registration::LlamaTestRegistration { name: "my_test[bar.gguf]", - key: ::llama_cpp_test_harness::load_key::LoadKey { - model_source: ::llama_cpp_test_harness::gguf_source::GgufSource::HuggingFace { + key: ::llama_cpp_test_model_provisioning::load_key::LoadKey { + model_source: ::llama_cpp_test_model_provisioning::gguf_source::GgufSource::HuggingFace { repo: "foo", file: "bar.gguf", }, mmproj_source: ::core::option::Option::None, - model_load_params: ::llama_cpp_test_harness::model_load_params::ModelLoadParams { + model_load_params: ::llama_cpp_test_model_provisioning::model_load_params::ModelLoadParams { n_gpu_layers: 0i32, load_mode: ::llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode::Mmap, }, @@ -247,6 +300,7 @@ mod tests { embeddings: false, }, func: ::llama_cpp_test_harness::llama_test_fn::LlamaTestFn::Fallible(my_test), + expected_outcome: ::llama_cpp_test_harness::expected_outcome::ExpectedOutcome::Success, } } }; @@ -254,6 +308,67 @@ mod tests { assert_eq!(expanded.to_string(), expected.to_string()); } + #[test] + fn a_setup_failure_expectation_emits_its_kind_and_reason() { + assert_eq!( + build_expected_outcome_literal(&ParsedExpectedOutcome::Failure( + ParsedExpectedFailure::PhaseSetup { + kind: Ident::new("ModelDownload", Span::call_site()), + reason: "the repository does not exist".to_owned(), + } + )) + .to_string(), + quote! { + ::llama_cpp_test_harness::expected_outcome::ExpectedOutcome::Failure { + failure: ::llama_cpp_test_harness::expected_failure::ExpectedFailure::PhaseSetup( + ::llama_cpp_test_model_provisioning::phase_setup_failure_kind::PhaseSetupFailureKind::ModelDownload + ), + reason: "the repository does not exist", + } + } + .to_string() + ); + } + + #[test] + fn a_test_failure_expectation_emits_its_reason() { + assert_eq!( + build_expected_outcome_literal(&ParsedExpectedOutcome::Failure( + ParsedExpectedFailure::Test { + reason: "the body bails on purpose".to_owned(), + } + )) + .to_string(), + quote! { + ::llama_cpp_test_harness::expected_outcome::ExpectedOutcome::Failure { + failure: ::llama_cpp_test_harness::expected_failure::ExpectedFailure::Test, + reason: "the body bails on purpose", + } + } + .to_string() + ); + } + + #[test] + fn a_test_failure_expected_of_a_test_that_cannot_return_an_error_is_refused() { + assert_eq!( + expansion_error( + quote! { + model_source = HuggingFace("foo", "bar.gguf"), + n_gpu_layers = 0, + load_mode = Mmap, + n_ctx = 1, + n_batch = 1, + n_ubatch = 1, + expected_failure = TestFailure("the body bails on purpose") + }, + quote! { fn my_test(fixture: &LlamaFixture<'_>) {} } + ), + "`expected_failure = TestFailure(...)` needs a test that returns a `Result`, because \ + only an error it returns counts as its failure" + ); + } + #[test] fn a_function_returning_a_result_registers_as_fallible() { assert_eq!( @@ -277,7 +392,7 @@ mod tests { assert_eq!( build_model_source_literal(&ParsedSource::LocalPath("/abs/local.gguf".to_owned())) .to_string(), - quote! { ::llama_cpp_test_harness::gguf_source::GgufSource::LocalPath("/abs/local.gguf") } + quote! { ::llama_cpp_test_model_provisioning::gguf_source::GgufSource::LocalPath("/abs/local.gguf") } .to_string() ); } @@ -291,7 +406,7 @@ mod tests { })) .to_string(), quote! { - ::core::option::Option::Some(::llama_cpp_test_harness::gguf_source::GgufSource::HuggingFace { + ::core::option::Option::Some(::llama_cpp_test_model_provisioning::gguf_source::GgufSource::HuggingFace { repo: "r", file: "mmproj.gguf", }) @@ -308,7 +423,7 @@ mod tests { ))) .to_string(), quote! { - ::core::option::Option::Some(::llama_cpp_test_harness::gguf_source::GgufSource::LocalPath( + ::core::option::Option::Some(::llama_cpp_test_model_provisioning::gguf_source::GgufSource::LocalPath( "/abs/mmproj.gguf" )) } @@ -321,7 +436,8 @@ mod tests { assert_eq!( expansion_error(quote! { totally_wrong = "x" }, well_formed_function()), "unknown field `totally_wrong`; expected one of: model_source, n_gpu_layers, load_mode, \ - n_ctx, n_batch, n_ubatch, mmproj_source, embeddings, n_seq_max, n_threads_batch" + n_ctx, n_batch, n_ubatch, mmproj_source, embeddings, n_seq_max, n_threads_batch, \ + expected_failure" ); } diff --git a/llama-cpp-test-harness-macros/src/lib.rs b/llama-cpp-test-harness-macros/src/lib.rs index 5a4e0ac1d..c58d222f0 100644 --- a/llama-cpp-test-harness-macros/src/lib.rs +++ b/llama-cpp-test-harness-macros/src/lib.rs @@ -7,6 +7,8 @@ mod attribute_accumulator; mod expand; mod parsed_args; mod parsed_context_params; +mod parsed_expected_failure; +mod parsed_expected_outcome; mod parsed_load_mode; mod parsed_model_load_params; mod parsed_source; @@ -43,7 +45,7 @@ mod tests { }; let message = "unknown field `totally_wrong`; expected one of: model_source, \ n_gpu_layers, load_mode, n_ctx, n_batch, n_ubatch, mmproj_source, \ - embeddings, n_seq_max, n_threads_batch"; + embeddings, n_seq_max, n_threads_batch, expected_failure"; assert_eq!( dispatch(attribute, item).to_string(), diff --git a/llama-cpp-test-harness-macros/src/parsed_args.rs b/llama-cpp-test-harness-macros/src/parsed_args.rs index 2ba617e3d..705b4d005 100644 --- a/llama-cpp-test-harness-macros/src/parsed_args.rs +++ b/llama-cpp-test-harness-macros/src/parsed_args.rs @@ -15,6 +15,8 @@ use syn::punctuated::Punctuated; use crate::attribute_accumulator::AttributeAccumulator; use crate::parsed_context_params::ParsedContextParams; +use crate::parsed_expected_failure::ParsedExpectedFailure; +use crate::parsed_expected_outcome::ParsedExpectedOutcome; use crate::parsed_load_mode::ParsedLoadMode; use crate::parsed_model_load_params::ParsedModelLoadParams; use crate::parsed_source::ParsedSource; @@ -33,6 +35,7 @@ const OPTIONAL_FIELDS: &[&str] = &[ "embeddings", "n_seq_max", "n_threads_batch", + "expected_failure", ]; fn literal_from_expression(expression: &Expr) -> syn::Result<&Lit> { @@ -189,6 +192,9 @@ fn dispatch_field( "n_threads_batch" => { accumulator.n_threads_batch = Some(require_i32_value(value, "n_threads_batch")?); } + "expected_failure" => { + accumulator.expected_failure = Some(ParsedExpectedFailure::parse(value)?); + } "repo" | "file" | "mmproj_file" => { return Err(syn::Error::new_spanned( identifier, @@ -217,6 +223,7 @@ pub struct ParsedArgs { pub mmproj_source: Option, pub model_load_params: ParsedModelLoadParams, pub context_params: ParsedContextParams, + pub expected_outcome: ParsedExpectedOutcome, } impl Parse for ParsedArgs { @@ -262,6 +269,10 @@ impl Parse for ParsedArgs { n_threads_batch: accumulator.n_threads_batch, embeddings: accumulator.embeddings.unwrap_or(false), }, + expected_outcome: accumulator.expected_failure.map_or( + ParsedExpectedOutcome::Success, + ParsedExpectedOutcome::Failure, + ), }) } } @@ -398,6 +409,19 @@ mod tests { ); } + #[test] + fn an_expected_failure_that_cannot_be_parsed_is_rejected() { + let source = format!("{ALL_REQUIRED}, expected_failure = Whatever(\"reason\")"); + + assert_eq!( + parse(&source) + .expect_err("an unknown expected failure must fail") + .to_string(), + "field `expected_failure` expects `SetupFailure(kind, \"reason\")` or \ + `TestFailure(\"reason\")`" + ); + } + #[test] fn unknown_field_is_rejected() { let source = format!("{ALL_REQUIRED}, surprise = 1"); @@ -408,7 +432,8 @@ mod tests { assert_eq!( message, "unknown field `surprise`; expected one of: model_source, n_gpu_layers, load_mode, \ - n_ctx, n_batch, n_ubatch, mmproj_source, embeddings, n_seq_max, n_threads_batch" + n_ctx, n_batch, n_ubatch, mmproj_source, embeddings, n_seq_max, n_threads_batch, \ + expected_failure" ); } diff --git a/llama-cpp-test-harness-macros/src/parsed_expected_failure.rs b/llama-cpp-test-harness-macros/src/parsed_expected_failure.rs new file mode 100644 index 000000000..250a88869 --- /dev/null +++ b/llama-cpp-test-harness-macros/src/parsed_expected_failure.rs @@ -0,0 +1,145 @@ +use proc_macro2::Ident; +use syn::Expr; +use syn::ExprCall; +use syn::ExprLit; +use syn::ExprPath; +use syn::Lit; + +const USAGE: &str = "field `expected_failure` expects `SetupFailure(kind, \"reason\")` or \ + `TestFailure(\"reason\")`"; + +fn reason_of(expression: &Expr) -> syn::Result { + if let Expr::Lit(ExprLit { + lit: Lit::Str(reason), + .. + }) = expression + && !reason.value().is_empty() + { + Ok(reason.value()) + } else { + Err(syn::Error::new_spanned( + expression, + "`expected_failure` expects a non-empty string stating why the trial fails on purpose", + )) + } +} + +fn setup_failure_kind_of(expression: &Expr) -> syn::Result { + if let Expr::Path(ExprPath { path, .. }) = expression + && let Some(kind) = path.get_ident() + { + Ok(kind.clone()) + } else { + Err(syn::Error::new_spanned( + expression, + "`SetupFailure` expects the bare name of a phase setup failure kind, such as \ + `ModelDownload`", + )) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ParsedExpectedFailure { + PhaseSetup { kind: Ident, reason: String }, + Test { reason: String }, +} + +impl ParsedExpectedFailure { + pub fn parse(expression: &Expr) -> syn::Result { + let Expr::Call(ExprCall { func, args, .. }) = expression else { + return Err(syn::Error::new_spanned(expression, USAGE)); + }; + let arguments: Vec<&Expr> = args.iter().collect(); + + match (func.as_ref(), arguments.as_slice()) { + (Expr::Path(ExprPath { path, .. }), [kind, reason]) + if path.is_ident("SetupFailure") => + { + Ok(Self::PhaseSetup { + kind: setup_failure_kind_of(kind)?, + reason: reason_of(reason)?, + }) + } + (Expr::Path(ExprPath { path, .. }), [reason]) if path.is_ident("TestFailure") => { + Ok(Self::Test { + reason: reason_of(reason)?, + }) + } + _ => Err(syn::Error::new_spanned(expression, USAGE)), + } + } +} + +#[cfg(test)] +mod tests { + use proc_macro2::Ident; + use proc_macro2::Span; + use syn::parse_str; + + use super::ParsedExpectedFailure; + + fn parse(source: &str) -> Result { + ParsedExpectedFailure::parse(&parse_str(source).expect("the source must be an expression")) + .map_err(|error| error.to_string()) + } + + #[test] + fn a_setup_failure_names_its_kind_and_reason() { + assert_eq!( + parse("SetupFailure(ModelDownload, \"the repository does not exist\")"), + Ok(ParsedExpectedFailure::PhaseSetup { + kind: Ident::new("ModelDownload", Span::call_site()), + reason: "the repository does not exist".to_owned(), + }) + ); + } + + #[test] + fn a_test_failure_names_its_reason() { + assert_eq!( + parse("TestFailure(\"the body bails on purpose\")"), + Ok(ParsedExpectedFailure::Test { + reason: "the body bails on purpose".to_owned(), + }) + ); + } + + #[test] + fn a_failure_that_is_not_a_call_is_refused() { + assert_eq!( + parse("ModelDownload"), + Err( + "field `expected_failure` expects `SetupFailure(kind, \"reason\")` or \ + `TestFailure(\"reason\")`" + .to_owned() + ) + ); + } + + #[test] + fn an_empty_reason_is_refused() { + assert_eq!( + [ + parse("SetupFailure(ModelDownload, \"\")"), + parse("TestFailure(\"\")") + ], + [(); 2].map(|()| Err( + "`expected_failure` expects a non-empty string stating why the trial fails on \ + purpose" + .to_owned() + )) + ); + } + + #[test] + fn a_setup_failure_kind_that_is_not_a_bare_name_is_refused() { + assert_eq!( + parse("SetupFailure(kinds::ModelDownload, \"reason\")"), + Err( + "`SetupFailure` expects the bare name of a phase setup failure kind, such as \ + `ModelDownload`" + .to_owned() + ) + ); + } +} diff --git a/llama-cpp-test-harness-macros/src/parsed_expected_outcome.rs b/llama-cpp-test-harness-macros/src/parsed_expected_outcome.rs new file mode 100644 index 000000000..e8b42d7c0 --- /dev/null +++ b/llama-cpp-test-harness-macros/src/parsed_expected_outcome.rs @@ -0,0 +1,7 @@ +use crate::parsed_expected_failure::ParsedExpectedFailure; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ParsedExpectedOutcome { + Success, + Failure(ParsedExpectedFailure), +} diff --git a/llama-cpp-test-harness/Cargo.toml b/llama-cpp-test-harness/Cargo.toml index d40cb3792..ad2ca129d 100644 --- a/llama-cpp-test-harness/Cargo.toml +++ b/llama-cpp-test-harness/Cargo.toml @@ -8,11 +8,11 @@ publish = false [dependencies] anyhow = { workspace = true } -hf-hub = { workspace = true } inventory = { workspace = true } libtest-mimic = { workspace = true } llama-cpp-bindings = { workspace = true } llama-cpp-test-harness-macros = { workspace = true } +llama-cpp-test-model-provisioning = { workspace = true } thiserror = { workspace = true } [features] diff --git a/llama-cpp-test-harness/src/download_model.rs b/llama-cpp-test-harness/src/download_model.rs deleted file mode 100644 index 80271612a..000000000 --- a/llama-cpp-test-harness/src/download_model.rs +++ /dev/null @@ -1,31 +0,0 @@ -use std::path::PathBuf; - -use anyhow::Result; -use hf_hub::api::sync::ApiBuilder; - -/// # Errors -/// -/// Returns an error if the HF client cannot be built or the file cannot be downloaded -/// (e.g., the repo or file does not exist, or network access fails). -pub fn download_model(repo: &str, file: &str) -> Result { - Ok(ApiBuilder::from_env() - .with_progress(true) - .build() - .and_then(|api| api.model(repo.to_owned()).get(file))?) -} - -#[cfg(test)] -mod tests { - use hf_hub::api::sync::ApiError; - - use super::download_model; - - #[test] - fn missing_file_in_real_repo_returns_error() { - let download_failure = - download_model("unsloth/Qwen3.5-0.8B-GGUF", "this-file-does-not-exist.gguf") - .expect_err("a file missing from an existing repository must fail to download"); - - assert!(download_failure.is::()); - } -} diff --git a/llama-cpp-test-harness/src/execution_phase.rs b/llama-cpp-test-harness/src/execution_phase.rs index 02fab4930..e52f68c52 100644 --- a/llama-cpp-test-harness/src/execution_phase.rs +++ b/llama-cpp-test-harness/src/execution_phase.rs @@ -1,5 +1,6 @@ +use llama_cpp_test_model_provisioning::load_key::LoadKey; + use crate::llama_test_registration::LlamaTestRegistration; -use crate::load_key::LoadKey; pub struct ExecutionPhase { pub key: LoadKey, @@ -27,11 +28,11 @@ impl ExecutionPhase { #[cfg(test)] mod tests { use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; + use llama_cpp_test_model_provisioning::gguf_source::GgufSource; + use llama_cpp_test_model_provisioning::load_key::LoadKey; + use llama_cpp_test_model_provisioning::model_load_params::ModelLoadParams; use super::ExecutionPhase; - use crate::gguf_source::GgufSource; - use crate::load_key::LoadKey; - use crate::model_load_params::ModelLoadParams; fn phase_with_source(source: GgufSource) -> ExecutionPhase { ExecutionPhase { diff --git a/llama-cpp-test-harness/src/execution_plan.rs b/llama-cpp-test-harness/src/execution_plan.rs index 08c2467b7..130b70452 100644 --- a/llama-cpp-test-harness/src/execution_plan.rs +++ b/llama-cpp-test-harness/src/execution_plan.rs @@ -68,14 +68,15 @@ impl ExecutionPlan { #[cfg(test)] mod tests { use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; + use llama_cpp_test_model_provisioning::gguf_source::GgufSource; + use llama_cpp_test_model_provisioning::load_key::LoadKey; + use llama_cpp_test_model_provisioning::model_load_params::ModelLoadParams; use super::ExecutionPlan; use crate::context_params::ContextParams; - use crate::gguf_source::GgufSource; + use crate::expected_outcome::ExpectedOutcome; use crate::llama_test_fn::LlamaTestFn; use crate::llama_test_registration::LlamaTestRegistration; - use crate::load_key::LoadKey; - use crate::model_load_params::ModelLoadParams; use crate::no_op::no_op; const TRIVIAL_CONTEXT_PARAMS: ContextParams = ContextParams { @@ -111,6 +112,7 @@ mod tests { }, context_params: TRIVIAL_CONTEXT_PARAMS, func: LlamaTestFn::Infallible(no_op), + expected_outcome: ExpectedOutcome::Success, }; static REG_BETA_B: LlamaTestRegistration = LlamaTestRegistration { name: "bravo", @@ -127,6 +129,7 @@ mod tests { }, context_params: TRIVIAL_CONTEXT_PARAMS, func: LlamaTestFn::Infallible(no_op), + expected_outcome: ExpectedOutcome::Success, }; static REG_ALPHA_Z: LlamaTestRegistration = LlamaTestRegistration { name: "zulu", @@ -143,6 +146,7 @@ mod tests { }, context_params: TRIVIAL_CONTEXT_PARAMS, func: LlamaTestFn::Infallible(no_op), + expected_outcome: ExpectedOutcome::Success, }; static REG_BETA_DIFFERENT_CONTEXT: LlamaTestRegistration = LlamaTestRegistration { name: "charlie", @@ -159,6 +163,7 @@ mod tests { }, context_params: ALTERNATE_CONTEXT_PARAMS, func: LlamaTestFn::Infallible(no_op), + expected_outcome: ExpectedOutcome::Success, }; #[test] diff --git a/llama-cpp-test-harness/src/expected_failure.rs b/llama-cpp-test-harness/src/expected_failure.rs new file mode 100644 index 000000000..6859d6e11 --- /dev/null +++ b/llama-cpp-test-harness/src/expected_failure.rs @@ -0,0 +1,37 @@ +use std::fmt; +use std::fmt::Display; +use std::fmt::Formatter; + +use llama_cpp_test_model_provisioning::phase_setup_failure_kind::PhaseSetupFailureKind; + +use crate::trial_outcome::TrialOutcome; + +/// The failure a trial declares it exists to produce. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExpectedFailure { + PhaseSetup(PhaseSetupFailureKind), + Test, +} + +impl ExpectedFailure { + #[must_use] + pub fn matches(self, outcome: &TrialOutcome<'_>) -> bool { + match (self, outcome) { + (Self::PhaseSetup(expected_kind), TrialOutcome::PhaseSetupFailed(setup_failure)) => { + setup_failure.kind() == expected_kind + } + (Self::Test, TrialOutcome::TestFailed(_)) => true, + (Self::PhaseSetup(_), TrialOutcome::Succeeded | TrialOutcome::TestFailed(_)) + | (Self::Test, TrialOutcome::Succeeded | TrialOutcome::PhaseSetupFailed(_)) => false, + } + } +} + +impl Display for ExpectedFailure { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::PhaseSetup(kind) => write!(formatter, "{kind}"), + Self::Test => formatter.write_str("TestFailure"), + } + } +} diff --git a/llama-cpp-test-harness/src/expected_outcome.rs b/llama-cpp-test-harness/src/expected_outcome.rs new file mode 100644 index 000000000..36a5bedb8 --- /dev/null +++ b/llama-cpp-test-harness/src/expected_outcome.rs @@ -0,0 +1,184 @@ +use libtest_mimic::Failed; +use libtest_mimic::Trial; + +use crate::expected_failure::ExpectedFailure; +use crate::trial_outcome::TrialOutcome; + +/// How a trial is declared to end: by succeeding, or by producing one exact failure on purpose, +/// for the stated reason. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExpectedOutcome { + Success, + Failure { + failure: ExpectedFailure, + reason: &'static str, + }, +} + +impl ExpectedOutcome { + /// Marks a trial that is declared to fail with the failure it expects and why. + #[must_use] + pub fn label(self, trial: Trial) -> Trial { + match self { + Self::Success => trial, + Self::Failure { failure, reason } => { + trial.with_kind(format!("expected {failure}: {reason}")) + } + } + } + + /// # Errors + /// + /// Returns [`Failed`] when `outcome` is not the declared one: a failure of a trial declared to + /// succeed, or a success or a different failure of a trial declared to fail. + pub fn judge(self, outcome: TrialOutcome<'_>) -> Result<(), Failed> { + match self { + Self::Success => match outcome { + TrialOutcome::Succeeded => Ok(()), + failure @ (TrialOutcome::PhaseSetupFailed(_) | TrialOutcome::TestFailed(_)) => { + Err(Failed::from(failure)) + } + }, + Self::Failure { failure, .. } if failure.matches(&outcome) => Ok(()), + Self::Failure { failure, .. } => { + Err(Failed::from(format!("expected {failure}, but {outcome}"))) + } + } + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use anyhow::anyhow; + + use llama_cpp_test_model_provisioning::phase_setup_error::PhaseSetupError; + use llama_cpp_test_model_provisioning::phase_setup_failure_kind::PhaseSetupFailureKind; + + use super::ExpectedOutcome; + use crate::expected_failure::ExpectedFailure; + use crate::trial_outcome::TrialOutcome; + + const EXPECTED_SETUP_FAILURE: ExpectedOutcome = ExpectedOutcome::Failure { + failure: ExpectedFailure::PhaseSetup(PhaseSetupFailureKind::MmprojPathNotUnicode), + reason: "the mmproj path is not Unicode on purpose", + }; + + const EXPECTED_TEST_FAILURE: ExpectedOutcome = ExpectedOutcome::Failure { + failure: ExpectedFailure::Test, + reason: "the body bails on purpose", + }; + + fn setup_failure() -> PhaseSetupError { + PhaseSetupError::MmprojPathNotUnicode { + mmproj_path: PathBuf::from("/models/mmproj.gguf"), + } + } + + fn verdict(expected_outcome: ExpectedOutcome, outcome: TrialOutcome<'_>) -> Result<(), String> { + expected_outcome + .judge(outcome) + .map_err(|failed| failed.message().unwrap_or_default().to_owned()) + } + + #[test] + fn a_success_meets_an_expected_success() { + assert_eq!( + verdict(ExpectedOutcome::Success, TrialOutcome::Succeeded), + Ok(()) + ); + } + + #[test] + fn a_failure_misses_an_expected_success_with_what_failed() { + let setup_failure = setup_failure(); + + assert_eq!( + [ + verdict( + ExpectedOutcome::Success, + TrialOutcome::PhaseSetupFailed(&setup_failure) + ), + verdict( + ExpectedOutcome::Success, + TrialOutcome::TestFailed(anyhow!("the body failed")) + ), + ], + [ + Err( + "phase setup failed: the mmproj path /models/mmproj.gguf is not Unicode" + .to_owned() + ), + Err("the body failed".to_owned()), + ] + ); + } + + #[test] + fn the_declared_failure_meets_its_expectation() { + let setup_failure = setup_failure(); + + assert_eq!( + [ + verdict( + EXPECTED_SETUP_FAILURE, + TrialOutcome::PhaseSetupFailed(&setup_failure) + ), + verdict( + EXPECTED_TEST_FAILURE, + TrialOutcome::TestFailed(anyhow!("the body bailed")) + ), + ], + [Ok(()), Ok(())] + ); + } + + #[test] + fn a_success_misses_an_expected_failure() { + assert_eq!( + verdict(EXPECTED_SETUP_FAILURE, TrialOutcome::Succeeded), + Err("expected MmprojPathNotUnicode, but the trial succeeded".to_owned()) + ); + } + + #[test] + fn a_different_failure_misses_an_expected_failure() { + let setup_failure = setup_failure(); + + assert_eq!( + [ + verdict( + ExpectedOutcome::Failure { + failure: ExpectedFailure::PhaseSetup(PhaseSetupFailureKind::ModelDownload), + reason: "the repository does not exist on purpose", + }, + TrialOutcome::PhaseSetupFailed(&setup_failure) + ), + verdict( + EXPECTED_SETUP_FAILURE, + TrialOutcome::TestFailed(anyhow!("the body failed")) + ), + verdict( + EXPECTED_TEST_FAILURE, + TrialOutcome::PhaseSetupFailed(&setup_failure) + ), + verdict(EXPECTED_TEST_FAILURE, TrialOutcome::Succeeded), + ], + [ + Err( + "expected ModelDownload, but phase setup failed: the mmproj path \ + /models/mmproj.gguf is not Unicode" + .to_owned() + ), + Err("expected MmprojPathNotUnicode, but the body failed".to_owned()), + Err( + "expected TestFailure, but phase setup failed: the mmproj path \ + /models/mmproj.gguf is not Unicode" + .to_owned() + ), + Err("expected TestFailure, but the trial succeeded".to_owned()), + ] + ); + } +} diff --git a/llama-cpp-test-harness/src/gguf_source.rs b/llama-cpp-test-harness/src/gguf_source.rs deleted file mode 100644 index 71d12cd58..000000000 --- a/llama-cpp-test-harness/src/gguf_source.rs +++ /dev/null @@ -1,72 +0,0 @@ -use std::fmt; -use std::fmt::Display; -use std::fmt::Formatter; -use std::path::PathBuf; - -use anyhow::Context; -use anyhow::Result; - -use crate::download_model::download_model; - -#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub enum GgufSource { - HuggingFace { - repo: &'static str, - file: &'static str, - }, - LocalPath(&'static str), -} - -impl Display for GgufSource { - fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { - match self { - Self::HuggingFace { repo, file } => write!(formatter, "{repo} / {file}"), - Self::LocalPath(path) => write!(formatter, "local:{path}"), - } - } -} - -impl GgufSource { - /// # Errors - /// - /// Returns an error if the HF download fails, with this source attached as its context. - /// `LocalPath` is infallible here — file existence is checked when the file is loaded. - pub fn resolve_path(self) -> Result { - match self { - Self::HuggingFace { repo, file } => download_model(repo, file).context(self), - Self::LocalPath(path) => Ok(PathBuf::from(path)), - } - } -} - -#[cfg(test)] -mod tests { - use std::path::PathBuf; - - use super::GgufSource; - - #[test] - fn resolve_path_for_local_path_returns_the_literal_path() { - let source = GgufSource::LocalPath("/abs/example.gguf"); - - let resolved = source - .resolve_path() - .expect("LocalPath resolve is infallible"); - - assert_eq!(resolved, PathBuf::from("/abs/example.gguf")); - } - - #[test] - fn resolve_path_failure_names_the_huggingface_source() { - let source = GgufSource::HuggingFace { - repo: "intentee-test-harness/does-not-exist", - file: "no-such-file.gguf", - }; - - let download_failure = source - .resolve_path() - .expect_err("a file from a repository that does not exist must fail to download"); - - assert_eq!(download_failure.downcast_ref::(), Some(&source)); - } -} diff --git a/llama-cpp-test-harness/src/harness_arguments_error.rs b/llama-cpp-test-harness/src/harness_arguments_error.rs deleted file mode 100644 index f73d4ab7d..000000000 --- a/llama-cpp-test-harness/src/harness_arguments_error.rs +++ /dev/null @@ -1,9 +0,0 @@ -use thiserror::Error; - -#[derive(Debug, Eq, Error, PartialEq)] -pub enum HarnessArgumentsError { - #[error( - "the test harness requires --test-threads=1 (or unset); got --test-threads={requested}" - )] - ConflictingTestThreads { requested: usize }, -} diff --git a/llama-cpp-test-harness/src/harness_run_error.rs b/llama-cpp-test-harness/src/harness_run_error.rs index b4d4d9e7e..58c5aa670 100644 --- a/llama-cpp-test-harness/src/harness_run_error.rs +++ b/llama-cpp-test-harness/src/harness_run_error.rs @@ -2,12 +2,12 @@ use thiserror::Error; use llama_cpp_bindings::error::llama_cpp_error::LlamaCppError; -use crate::harness_arguments_error::HarnessArgumentsError; - #[derive(Debug, Eq, Error, PartialEq)] pub enum HarnessRunError { - #[error("failed to parse harness arguments: {0}")] - ArgumentParsing(#[from] HarnessArgumentsError), + #[error( + "the test harness requires --test-threads=1 (or unset); got --test-threads={requested}" + )] + ConflictingTestThreads { requested: usize }, #[error("failed to initialise the llama backend: {0}")] BackendInit(#[from] LlamaCppError), } diff --git a/llama-cpp-test-harness/src/lib.rs b/llama-cpp-test-harness/src/lib.rs index c41df52fb..fbc107a0f 100644 --- a/llama-cpp-test-harness/src/lib.rs +++ b/llama-cpp-test-harness/src/lib.rs @@ -4,25 +4,18 @@ )] pub mod context_params; -pub mod download_model; pub mod execution_phase; pub mod execution_plan; -pub mod gguf_source; -pub mod harness_arguments_error; +pub mod expected_failure; +pub mod expected_outcome; pub mod harness_run_error; pub mod llama_fixture; pub mod llama_test_fn; pub mod llama_test_registration; pub mod llama_tests_main_macro; -pub mod load_key; -pub mod model_load_params; pub mod no_op; pub mod parse_harness_arguments; -pub mod phase_model_files; -pub mod phase_setup; -pub mod phase_state; pub mod run; pub mod run_to_conclusions; pub mod scheduled_phase; -#[cfg(test)] -mod test_backend_gate; +pub mod trial_outcome; diff --git a/llama-cpp-test-harness/src/llama_test_registration.rs b/llama-cpp-test-harness/src/llama_test_registration.rs index 1d6ce5e7b..5720e1a78 100644 --- a/llama-cpp-test-harness/src/llama_test_registration.rs +++ b/llama-cpp-test-harness/src/llama_test_registration.rs @@ -1,14 +1,17 @@ use inventory::collect; +use llama_cpp_test_model_provisioning::load_key::LoadKey; + use crate::context_params::ContextParams; +use crate::expected_outcome::ExpectedOutcome; use crate::llama_test_fn::LlamaTestFn; -use crate::load_key::LoadKey; pub struct LlamaTestRegistration { pub name: &'static str, pub key: LoadKey, pub context_params: ContextParams, pub func: LlamaTestFn, + pub expected_outcome: ExpectedOutcome, } collect!(LlamaTestRegistration); diff --git a/llama-cpp-test-harness/src/parse_harness_arguments.rs b/llama-cpp-test-harness/src/parse_harness_arguments.rs index bf761c830..1168ff879 100644 --- a/llama-cpp-test-harness/src/parse_harness_arguments.rs +++ b/llama-cpp-test-harness/src/parse_harness_arguments.rs @@ -1,23 +1,23 @@ use libtest_mimic::Arguments; -use crate::harness_arguments_error::HarnessArgumentsError; +use crate::harness_run_error::HarnessRunError; -fn validate(mut arguments: Arguments) -> Result { +fn validate(mut arguments: Arguments) -> Result { match arguments.test_threads { None | Some(1) => { arguments.test_threads = Some(1); Ok(arguments) } - Some(requested) => Err(HarnessArgumentsError::ConflictingTestThreads { requested }), + Some(requested) => Err(HarnessRunError::ConflictingTestThreads { requested }), } } /// # Errors /// -/// Returns [`HarnessArgumentsError::ConflictingTestThreads`] when `--test-threads` is set to +/// Returns [`HarnessRunError::ConflictingTestThreads`] when `--test-threads` is set to /// any value other than `1`. The harness orchestrates phase batching itself and cannot share /// that responsibility with `libtest_mimic`'s thread pool. -pub fn parse_harness_arguments(arguments: Arguments) -> Result { +pub fn parse_harness_arguments(arguments: Arguments) -> Result { validate(arguments) } @@ -26,7 +26,7 @@ mod tests { use libtest_mimic::Arguments; use super::validate; - use crate::harness_arguments_error::HarnessArgumentsError; + use crate::harness_run_error::HarnessRunError; #[test] fn validate_accepts_unset_test_threads_and_defaults_to_one() { @@ -57,7 +57,7 @@ mod tests { assert_eq!( error, - HarnessArgumentsError::ConflictingTestThreads { requested: 8 } + HarnessRunError::ConflictingTestThreads { requested: 8 } ); } diff --git a/llama-cpp-test-harness/src/phase_model_files.rs b/llama-cpp-test-harness/src/phase_model_files.rs deleted file mode 100644 index 4b1182c07..000000000 --- a/llama-cpp-test-harness/src/phase_model_files.rs +++ /dev/null @@ -1,101 +0,0 @@ -use std::path::PathBuf; -use std::sync::Arc; - -use anyhow::Result; - -use llama_cpp_bindings::llama_backend::LlamaBackend; -use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::mtmd::mtmd_context::MtmdContext; -use llama_cpp_bindings::mtmd::mtmd_context_params::MtmdContextParams; - -use crate::model_load_params::ModelLoadParams; -use crate::phase_state::PhaseState; - -#[derive(Debug)] -pub struct PhaseModelFiles { - pub model_path: PathBuf, - pub mmproj_path: Option, -} - -impl PhaseModelFiles { - /// # Errors - /// - /// Returns an error if loading the model into llama.cpp fails, or initializing the - /// `MtmdContext` from the mmproj file fails. - pub fn load_phase_state( - &self, - backend: &Arc, - model_load_params: ModelLoadParams, - ) -> Result { - let Self { - model_path, - mmproj_path, - } = self; - let model_params = model_load_params.into_llama_model_params(); - let model = LlamaModel::load_from_file(backend, model_path, &model_params)?; - - let mtmd_context = match mmproj_path { - Some(mmproj_path) => { - let mmproj_path_str = mmproj_path.to_string_lossy(); - let params = MtmdContextParams::default(); - Some(MtmdContext::init_from_file( - mmproj_path_str.as_ref(), - &model, - ¶ms, - )?) - } - None => None, - }; - - Ok(PhaseState { - mtmd_context, - model, - backend: Arc::clone(backend), - model_path: model_path.clone(), - }) - } -} - -#[cfg(test)] -mod tests { - use std::path::PathBuf; - use std::sync::Arc; - use std::sync::PoisonError; - - use llama_cpp_bindings::error::llama_model_load_error::LlamaModelLoadError; - use llama_cpp_bindings::llama_backend::LlamaBackend; - use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; - - use super::PhaseModelFiles; - use crate::model_load_params::ModelLoadParams; - use crate::test_backend_gate::BACKEND_INIT_GATE; - - const NON_GGUF_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/Cargo.toml"); - - #[test] - fn load_phase_state_propagates_model_load_failure() { - let _gate = BACKEND_INIT_GATE - .lock() - .unwrap_or_else(PoisonError::into_inner); - let backend = Arc::new(LlamaBackend::init().expect("backend init must succeed")); - let model_files = PhaseModelFiles { - model_path: PathBuf::from(NON_GGUF_PATH), - mmproj_path: None, - }; - - let load_failure = model_files - .load_phase_state( - &backend, - ModelLoadParams { - n_gpu_layers: 0, - load_mode: LlamaLoadMode::Mmap, - }, - ) - .expect_err("a model file that is not GGUF must fail to load"); - - assert_eq!( - load_failure.downcast_ref::(), - Some(&LlamaModelLoadError::Unloadable) - ); - } -} diff --git a/llama-cpp-test-harness/src/run.rs b/llama-cpp-test-harness/src/run.rs index 57d0db565..3d97e9998 100644 --- a/llama-cpp-test-harness/src/run.rs +++ b/llama-cpp-test-harness/src/run.rs @@ -32,10 +32,10 @@ mod tests { use libtest_mimic::Conclusion; use llama_cpp_bindings::llama_backend::LlamaBackend; + use llama_cpp_test_model_provisioning::backend_init_gate::BACKEND_INIT_GATE; use super::aggregate_exit_code; use super::run; - use crate::test_backend_gate::BACKEND_INIT_GATE; fn passing_conclusion() -> Conclusion { Conclusion { diff --git a/llama-cpp-test-harness/src/run_to_conclusions.rs b/llama-cpp-test-harness/src/run_to_conclusions.rs index 86e1f24a3..ca64febce 100644 --- a/llama-cpp-test-harness/src/run_to_conclusions.rs +++ b/llama-cpp-test-harness/src/run_to_conclusions.rs @@ -30,11 +30,10 @@ mod tests { use llama_cpp_bindings::error::llama_cpp_error::LlamaCppError; use llama_cpp_bindings::llama_backend::LlamaBackend; + use llama_cpp_test_model_provisioning::backend_init_gate::BACKEND_INIT_GATE; use super::run_to_conclusions; - use crate::harness_arguments_error::HarnessArgumentsError; use crate::harness_run_error::HarnessRunError; - use crate::test_backend_gate::BACKEND_INIT_GATE; #[test] fn conflicting_arguments_stop_the_run_before_the_backend_starts() { @@ -44,9 +43,7 @@ mod tests { ..Arguments::default() }) .err(), - Some(HarnessRunError::ArgumentParsing( - HarnessArgumentsError::ConflictingTestThreads { requested: 8 } - )) + Some(HarnessRunError::ConflictingTestThreads { requested: 8 }) ); } diff --git a/llama-cpp-test-harness/src/scheduled_phase.rs b/llama-cpp-test-harness/src/scheduled_phase.rs index 9644061fc..da71ea4f4 100644 --- a/llama-cpp-test-harness/src/scheduled_phase.rs +++ b/llama-cpp-test-harness/src/scheduled_phase.rs @@ -2,34 +2,42 @@ use std::sync::Arc; use libtest_mimic::Arguments; use libtest_mimic::Conclusion; -use libtest_mimic::Failed; use libtest_mimic::Trial; use llama_cpp_bindings::llama_backend::LlamaBackend; +use llama_cpp_test_model_provisioning::phase_setup::PhaseSetup; use crate::execution_phase::ExecutionPhase; use crate::llama_fixture::LlamaFixture; use crate::llama_test_registration::LlamaTestRegistration; -use crate::phase_setup::PhaseSetup; +use crate::trial_outcome::TrialOutcome; + +fn trial_outcome<'setup>( + registration: &'static LlamaTestRegistration, + setup: &'setup PhaseSetup, +) -> TrialOutcome<'setup> { + match setup.load_phase_state() { + Ok(state) => match registration.func.run(&LlamaFixture { + model: &state.model, + backend: &state.backend, + context_params: ®istration.context_params, + mtmd_context: state.mtmd_context.as_ref(), + model_path: &state.model_path, + }) { + Ok(()) => TrialOutcome::Succeeded, + Err(test_failure) => TrialOutcome::TestFailed(test_failure), + }, + Err(setup_failure) => TrialOutcome::PhaseSetupFailed(setup_failure), + } +} fn trial(registration: &'static LlamaTestRegistration, setup: &Arc) -> Trial { let setup = Arc::clone(setup); + let expected_outcome = registration.expected_outcome; - Trial::test(registration.name, move || match setup.load_phase_state() { - Ok(state) => registration - .func - .run(&LlamaFixture { - model: &state.model, - backend: &state.backend, - context_params: ®istration.context_params, - mtmd_context: state.mtmd_context.as_ref(), - model_path: &state.model_path, - }) - .map_err(|error| Failed::from(format!("{error:#}"))), - Err(setup_failure) => Err(Failed::from(format!( - "phase setup failed: {setup_failure:#}" - ))), - }) + expected_outcome.label(Trial::test(registration.name, move || { + expected_outcome.judge(trial_outcome(registration, &setup)) + })) } pub struct ScheduledPhase<'plan> { @@ -95,52 +103,101 @@ mod tests { use llama_cpp_bindings::llama_backend::LlamaBackend; use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; + use llama_cpp_test_model_provisioning::backend_init_gate::BACKEND_INIT_GATE; + use llama_cpp_test_model_provisioning::gguf_source::GgufSource; + use llama_cpp_test_model_provisioning::load_key::LoadKey; + use llama_cpp_test_model_provisioning::model_load_params::ModelLoadParams; + use llama_cpp_test_model_provisioning::phase_setup_failure_kind::PhaseSetupFailureKind; use super::ScheduledPhase; use crate::context_params::ContextParams; use crate::execution_phase::ExecutionPhase; - use crate::gguf_source::GgufSource; + use crate::expected_failure::ExpectedFailure; + use crate::expected_outcome::ExpectedOutcome; use crate::llama_test_fn::LlamaTestFn; use crate::llama_test_registration::LlamaTestRegistration; - use crate::load_key::LoadKey; - use crate::model_load_params::ModelLoadParams; use crate::no_op::no_op; - use crate::test_backend_gate::BACKEND_INIT_GATE; + + const LOAD_KEY: LoadKey = LoadKey { + model_source: GgufSource::LocalPath("/nonexistent/llama-cpp-test-harness/model.gguf"), + mmproj_source: None, + model_load_params: ModelLoadParams { + n_gpu_layers: 0, + load_mode: LlamaLoadMode::Mmap, + }, + }; + + const CONTEXT_PARAMS: ContextParams = ContextParams { + n_ctx: 1, + n_batch: 1, + n_ubatch: 1, + n_seq_max: 1, + n_threads_batch: None, + embeddings: false, + }; static REGISTRATION: LlamaTestRegistration = LlamaTestRegistration { name: "scheduled_trial", - key: LoadKey { - model_source: GgufSource::LocalPath("/nonexistent/llama-cpp-test-harness/model.gguf"), - mmproj_source: None, - model_load_params: ModelLoadParams { - n_gpu_layers: 0, - load_mode: LlamaLoadMode::Mmap, - }, - }, - context_params: ContextParams { - n_ctx: 1, - n_batch: 1, - n_ubatch: 1, - n_seq_max: 1, - n_threads_batch: None, - embeddings: false, - }, + key: LOAD_KEY, + context_params: CONTEXT_PARAMS, func: LlamaTestFn::Infallible(no_op), + expected_outcome: ExpectedOutcome::Success, }; - fn selects_any_trial_with(harness_arguments: &[&str]) -> bool { + static FAILING_ON_PURPOSE_REGISTRATION: LlamaTestRegistration = LlamaTestRegistration { + name: "failing_on_purpose", + key: LOAD_KEY, + context_params: CONTEXT_PARAMS, + func: LlamaTestFn::Infallible(no_op), + expected_outcome: ExpectedOutcome::Failure { + failure: ExpectedFailure::PhaseSetup(PhaseSetupFailureKind::ModelLoad), + reason: "the model file does not exist on purpose", + }, + }; + + fn with_scheduled_phase) -> TResult>( + registrations: Vec<&'static LlamaTestRegistration>, + use_scheduled_phase: TUse, + ) -> TResult { let _gate = BACKEND_INIT_GATE .lock() .unwrap_or_else(PoisonError::into_inner); let backend = Arc::new(LlamaBackend::init().expect("backend init must succeed")); let phase = ExecutionPhase { - key: REGISTRATION.key, - registrations: vec![®ISTRATION], + key: LOAD_KEY, + registrations, }; + + use_scheduled_phase(&ScheduledPhase::new(&phase, &backend)) + } + + fn selects_any_trial_with(harness_arguments: &[&str]) -> bool { let arguments = Arguments::from_iter(iter::once("harness").chain(harness_arguments.iter().copied())); - ScheduledPhase::new(&phase, &backend).selects_any_trial(&arguments) + with_scheduled_phase(vec![®ISTRATION], |scheduled_phase| { + scheduled_phase.selects_any_trial(&arguments) + }) + } + + #[test] + fn only_a_trial_declared_to_fail_is_labelled_with_the_failure_it_expects_and_why() { + assert_eq!( + with_scheduled_phase( + vec![®ISTRATION, &FAILING_ON_PURPOSE_REGISTRATION], + |scheduled_phase| { + scheduled_phase + .trials + .iter() + .map(|trial| trial.kind().to_owned()) + .collect::>() + } + ), + [ + String::new(), + "expected ModelLoad: the model file does not exist on purpose".to_owned() + ] + ); } #[test] diff --git a/llama-cpp-test-harness/src/test_backend_gate.rs b/llama-cpp-test-harness/src/test_backend_gate.rs deleted file mode 100644 index 4809dabd5..000000000 --- a/llama-cpp-test-harness/src/test_backend_gate.rs +++ /dev/null @@ -1,4 +0,0 @@ -use std::sync::Mutex; - -#[cfg(test)] -pub static BACKEND_INIT_GATE: Mutex<()> = Mutex::new(()); diff --git a/llama-cpp-test-harness/src/trial_outcome.rs b/llama-cpp-test-harness/src/trial_outcome.rs new file mode 100644 index 000000000..48a07511f --- /dev/null +++ b/llama-cpp-test-harness/src/trial_outcome.rs @@ -0,0 +1,27 @@ +use std::fmt; +use std::fmt::Display; +use std::fmt::Formatter; + +use anyhow::Error; + +use llama_cpp_test_model_provisioning::phase_setup_error::PhaseSetupError; + +/// How a trial actually ended. +#[derive(Debug)] +pub enum TrialOutcome<'setup> { + Succeeded, + PhaseSetupFailed(&'setup PhaseSetupError), + TestFailed(Error), +} + +impl Display for TrialOutcome<'_> { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::Succeeded => formatter.write_str("the trial succeeded"), + Self::PhaseSetupFailed(setup_failure) => { + write!(formatter, "phase setup failed: {setup_failure}") + } + Self::TestFailed(test_failure) => write!(formatter, "{test_failure:#}"), + } + } +} diff --git a/llama-cpp-test-harness/tests/phase_conclusions.rs b/llama-cpp-test-harness/tests/phase_conclusions.rs index 2dea0f8a8..014c92f29 100644 --- a/llama-cpp-test-harness/tests/phase_conclusions.rs +++ b/llama-cpp-test-harness/tests/phase_conclusions.rs @@ -39,7 +39,8 @@ fn phase_a_second_passing_trial(fixture: &LlamaFixture<'_>) { load_mode = Mmap, n_ctx = 512, n_batch = 128, - n_ubatch = 64 + n_ubatch = 64, + expected_failure = TestFailure("the body bails to exercise reporting a failing test"), )] fn phase_a_intentionally_failing_trial(_fixture: &LlamaFixture<'_>) -> Result<()> { bail!("intentional failure to exercise the trial-failure dispatch path"); @@ -83,7 +84,11 @@ fn phase_b_second_passing_trial(fixture: &LlamaFixture<'_>) { load_mode = Mmap, n_ctx = 1, n_batch = 1, - n_ubatch = 1 + n_ubatch = 1, + expected_failure = SetupFailure( + ModelDownload, + "the model repository does not exist, so the phase cannot download its model" + ), )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), @@ -93,6 +98,10 @@ fn phase_b_second_passing_trial(fixture: &LlamaFixture<'_>) { n_batch = 128, n_ubatch = 64, mmproj_source = HuggingFace("intentee-test-harness/does-not-exist", "no-such-mmproj.gguf"), + expected_failure = SetupFailure( + MmprojDownload, + "the mmproj repository does not exist, so the phase cannot download its mmproj" + ), )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), @@ -102,6 +111,10 @@ fn phase_b_second_passing_trial(fixture: &LlamaFixture<'_>) { n_batch = 128, n_ubatch = 64, mmproj_source = LocalPath("/nonexistent/llama-cpp-test-harness/no-such-mmproj.gguf"), + expected_failure = SetupFailure( + MmprojLoad, + "the local mmproj file does not exist, so llama.cpp cannot load it" + ), )] #[llama_test( model_source = LocalPath("Cargo.toml"), @@ -109,7 +122,11 @@ fn phase_b_second_passing_trial(fixture: &LlamaFixture<'_>) { load_mode = Mmap, n_ctx = 1, n_batch = 1, - n_ubatch = 1 + n_ubatch = 1, + expected_failure = SetupFailure( + ModelLoad, + "Cargo.toml is not a GGUF model, so llama.cpp cannot load it" + ), )] fn phase_a_trial_also_scheduled_into_every_setup_failure_phase(fixture: &LlamaFixture<'_>) { assert!(fixture.model_path.exists()); @@ -132,8 +149,6 @@ fn phase_d_passing_trial_with_a_loaded_mmproj(fixture: &LlamaFixture<'_>) { } const EXPECTED_PHASES: usize = 7; -const EXPECTED_PASSED: u64 = 6; -const EXPECTED_FAILED: u64 = 5; fn main() -> ExitCode { let conclusions = match run_to_conclusions(Arguments::from_args()) { @@ -145,29 +160,19 @@ fn main() -> ExitCode { } }; let phases = conclusions.len(); - let total_passed: u64 = conclusions - .iter() - .map(|conclusion| conclusion.num_passed) - .sum(); - let total_failed: u64 = conclusions + let phases_with_a_failed_trial = conclusions .iter() - .map(|conclusion| conclusion.num_failed) - .sum(); + .filter(|conclusion| conclusion.has_failed()) + .count(); - if phases == EXPECTED_PHASES - && total_passed == EXPECTED_PASSED - && total_failed == EXPECTED_FAILED - { - eprintln!( - "phase_conclusions: as expected — phases={phases}, passed={total_passed}, failed={total_failed}" - ); + if phases == EXPECTED_PHASES && phases_with_a_failed_trial == 0 { ExitCode::SUCCESS } else { eprintln!( - "phase_conclusions: UNEXPECTED — phases={phases} (want {EXPECTED_PHASES}), \ - passed={total_passed} (want {EXPECTED_PASSED}), \ - failed={total_failed} (want {EXPECTED_FAILED})" + "phase_conclusions: want {EXPECTED_PHASES} phases without a failed trial; got \ + {phases} phases, {phases_with_a_failed_trial} of them with a failed trial" ); + ExitCode::FAILURE } } diff --git a/llama-cpp-test-model-provisioning/Cargo.toml b/llama-cpp-test-model-provisioning/Cargo.toml new file mode 100644 index 000000000..074cd44c6 --- /dev/null +++ b/llama-cpp-test-model-provisioning/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "llama-cpp-test-model-provisioning" +description = "Downloads and loads the models llama-cpp-bindings integration tests run against" +version.workspace = true +edition.workspace = true +license.workspace = true +publish = false + +[dependencies] +hf-hub = { workspace = true } +llama-cpp-bindings = { workspace = true } +thiserror = { workspace = true } + +[features] +cuda = ["llama-cpp-bindings/cuda"] +cuda-no-vmm = ["llama-cpp-bindings/cuda-no-vmm"] +metal = ["llama-cpp-bindings/metal"] +vulkan = ["llama-cpp-bindings/vulkan"] +rocm = ["llama-cpp-bindings/rocm"] + +[lints] +workspace = true diff --git a/llama-cpp-test-model-provisioning/src/backend_init_gate.rs b/llama-cpp-test-model-provisioning/src/backend_init_gate.rs new file mode 100644 index 000000000..61f6765c4 --- /dev/null +++ b/llama-cpp-test-model-provisioning/src/backend_init_gate.rs @@ -0,0 +1,5 @@ +use std::sync::Mutex; + +/// Serializes the tests of one process that initialize the llama backend, which llama.cpp allows +/// only once at a time. +pub static BACKEND_INIT_GATE: Mutex<()> = Mutex::new(()); diff --git a/llama-cpp-test-model-provisioning/src/download_model.rs b/llama-cpp-test-model-provisioning/src/download_model.rs new file mode 100644 index 000000000..c88d03833 --- /dev/null +++ b/llama-cpp-test-model-provisioning/src/download_model.rs @@ -0,0 +1,15 @@ +use std::path::PathBuf; + +use hf_hub::api::sync::ApiBuilder; +use hf_hub::api::sync::ApiError; + +/// # Errors +/// +/// Returns the [`ApiError`] of building the HF client or of downloading the file, e.g. when the +/// repository or file does not exist, or network access fails. +pub fn download_model(repo: &str, file: &str) -> Result { + ApiBuilder::from_env() + .with_progress(true) + .build() + .and_then(|api| api.model(repo.to_owned()).get(file)) +} diff --git a/llama-cpp-test-model-provisioning/src/gguf_source.rs b/llama-cpp-test-model-provisioning/src/gguf_source.rs new file mode 100644 index 000000000..3cb2be25a --- /dev/null +++ b/llama-cpp-test-model-provisioning/src/gguf_source.rs @@ -0,0 +1,39 @@ +use std::fmt; +use std::fmt::Display; +use std::fmt::Formatter; +use std::path::PathBuf; + +use hf_hub::api::sync::ApiError; + +use crate::download_model::download_model; + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub enum GgufSource { + HuggingFace { + repo: &'static str, + file: &'static str, + }, + LocalPath(&'static str), +} + +impl Display for GgufSource { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::HuggingFace { repo, file } => write!(formatter, "{repo} / {file}"), + Self::LocalPath(path) => write!(formatter, "local:{path}"), + } + } +} + +impl GgufSource { + /// # Errors + /// + /// Returns the [`ApiError`] of downloading a `HuggingFace` source. `LocalPath` is infallible + /// here; whether the file exists is checked when it is loaded. + pub fn resolve_path(self) -> Result { + match self { + Self::HuggingFace { repo, file } => download_model(repo, file), + Self::LocalPath(path) => Ok(PathBuf::from(path)), + } + } +} diff --git a/llama-cpp-test-model-provisioning/src/lib.rs b/llama-cpp-test-model-provisioning/src/lib.rs new file mode 100644 index 000000000..9ecbfeca9 --- /dev/null +++ b/llama-cpp-test-model-provisioning/src/lib.rs @@ -0,0 +1,15 @@ +#![cfg_attr( + not(test), + deny(clippy::unwrap_used, clippy::expect_used, clippy::panic) +)] + +pub mod backend_init_gate; +pub mod download_model; +pub mod gguf_source; +pub mod load_key; +pub mod model_load_params; +pub mod phase_model_files; +pub mod phase_setup; +pub mod phase_setup_error; +pub mod phase_setup_failure_kind; +pub mod phase_state; diff --git a/llama-cpp-test-harness/src/load_key.rs b/llama-cpp-test-model-provisioning/src/load_key.rs similarity index 63% rename from llama-cpp-test-harness/src/load_key.rs rename to llama-cpp-test-model-provisioning/src/load_key.rs index 2ae9f9ff0..964f070f9 100644 --- a/llama-cpp-test-harness/src/load_key.rs +++ b/llama-cpp-test-model-provisioning/src/load_key.rs @@ -1,8 +1,7 @@ -use anyhow::Result; - use crate::gguf_source::GgufSource; use crate::model_load_params::ModelLoadParams; use crate::phase_model_files::PhaseModelFiles; +use crate::phase_setup_error::PhaseSetupError; #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub struct LoadKey { @@ -14,13 +13,31 @@ pub struct LoadKey { impl LoadKey { /// # Errors /// - /// Returns an error if downloading the model file or the mmproj file fails. - pub fn download_model_files(&self) -> Result { + /// Returns [`PhaseSetupError::ModelDownload`] or [`PhaseSetupError::MmprojDownload`] when the + /// model or the mmproj file cannot be downloaded. + pub fn download_model_files(&self) -> Result { + let Self { + model_source, + mmproj_source, + .. + } = *self; + Ok(PhaseModelFiles { - model_path: self.model_source.resolve_path()?, - mmproj_path: self - .mmproj_source - .map(GgufSource::resolve_path) + model_path: model_source.resolve_path().map_err(|cause| { + PhaseSetupError::ModelDownload { + model_source, + cause, + } + })?, + mmproj_path: mmproj_source + .map(|mmproj_source| { + mmproj_source + .resolve_path() + .map_err(|cause| PhaseSetupError::MmprojDownload { + mmproj_source, + cause, + }) + }) .transpose()?, }) } @@ -35,6 +52,7 @@ mod tests { use super::LoadKey; use crate::gguf_source::GgufSource; use crate::model_load_params::ModelLoadParams; + use crate::phase_setup_failure_kind::PhaseSetupFailureKind; fn baseline() -> LoadKey { LoadKey { @@ -111,22 +129,35 @@ mod tests { } #[test] - fn download_model_files_propagates_mmproj_download_failure() { - let missing_mmproj_source = GgufSource::HuggingFace { + fn a_model_that_cannot_be_downloaded_fails_as_a_model_download() { + let mut key = baseline(); + key.model_source = GgufSource::HuggingFace { repo: "intentee-test-harness/does-not-exist", - file: "no-such-mmproj.gguf", + file: "no-such-file.gguf", }; + + assert_eq!( + key.download_model_files() + .err() + .map(|setup_failure| setup_failure.kind()), + Some(PhaseSetupFailureKind::ModelDownload) + ); + } + + #[test] + fn an_mmproj_that_cannot_be_downloaded_fails_as_an_mmproj_download() { let mut key = baseline(); key.model_source = GgufSource::LocalPath("/abs/model.gguf"); - key.mmproj_source = Some(missing_mmproj_source); - - let download_failure = key - .download_model_files() - .expect_err("an mmproj from a repository that does not exist must fail to download"); + key.mmproj_source = Some(GgufSource::HuggingFace { + repo: "intentee-test-harness/does-not-exist", + file: "no-such-mmproj.gguf", + }); assert_eq!( - download_failure.downcast_ref::(), - Some(&missing_mmproj_source) + key.download_model_files() + .err() + .map(|setup_failure| setup_failure.kind()), + Some(PhaseSetupFailureKind::MmprojDownload) ); } } diff --git a/llama-cpp-test-harness/src/model_load_params.rs b/llama-cpp-test-model-provisioning/src/model_load_params.rs similarity index 100% rename from llama-cpp-test-harness/src/model_load_params.rs rename to llama-cpp-test-model-provisioning/src/model_load_params.rs diff --git a/llama-cpp-test-model-provisioning/src/phase_model_files.rs b/llama-cpp-test-model-provisioning/src/phase_model_files.rs new file mode 100644 index 000000000..848157e06 --- /dev/null +++ b/llama-cpp-test-model-provisioning/src/phase_model_files.rs @@ -0,0 +1,136 @@ +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + +use llama_cpp_bindings::llama_backend::LlamaBackend; +use llama_cpp_bindings::model::LlamaModel; +use llama_cpp_bindings::mtmd::mtmd_context::MtmdContext; +use llama_cpp_bindings::mtmd::mtmd_context_params::MtmdContextParams; + +use crate::model_load_params::ModelLoadParams; +use crate::phase_setup_error::PhaseSetupError; +use crate::phase_state::PhaseState; + +fn unicode_mmproj_path(mmproj_path: &Path) -> Result<&str, PhaseSetupError> { + mmproj_path + .to_str() + .ok_or_else(|| PhaseSetupError::MmprojPathNotUnicode { + mmproj_path: mmproj_path.to_path_buf(), + }) +} + +#[derive(Debug)] +pub struct PhaseModelFiles { + pub model_path: PathBuf, + pub mmproj_path: Option, +} + +impl PhaseModelFiles { + /// # Errors + /// + /// Returns [`PhaseSetupError::MmprojPathNotUnicode`] before loading anything when the mmproj + /// path cannot be handed to llama.cpp, [`PhaseSetupError::ModelLoad`] when llama.cpp cannot + /// load the model, or [`PhaseSetupError::MmprojLoad`] when it cannot load the mmproj. + pub fn load_phase_state( + &self, + backend: &Arc, + model_load_params: ModelLoadParams, + ) -> Result { + let Self { + model_path, + mmproj_path, + } = self; + let unicode_mmproj_path = mmproj_path + .as_deref() + .map(unicode_mmproj_path) + .transpose()?; + let model = LlamaModel::load_from_file( + backend, + model_path, + &model_load_params.into_llama_model_params(), + ) + .map_err(|cause| PhaseSetupError::ModelLoad { + model_path: model_path.clone(), + cause, + })?; + let mtmd_context = unicode_mmproj_path + .map(|unicode_mmproj_path| { + MtmdContext::init_from_file( + unicode_mmproj_path, + &model, + &MtmdContextParams::default(), + ) + .map_err(|cause| PhaseSetupError::MmprojLoad { + mmproj_path: PathBuf::from(unicode_mmproj_path), + cause, + }) + }) + .transpose()?; + + Ok(PhaseState { + mtmd_context, + model, + backend: Arc::clone(backend), + model_path: model_path.clone(), + }) + } +} + +#[cfg(test)] +mod tests { + use std::ffi::OsString; + use std::os::unix::ffi::OsStringExt; + use std::path::PathBuf; + use std::sync::Arc; + use std::sync::PoisonError; + + use llama_cpp_bindings::llama_backend::LlamaBackend; + use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; + + use super::PhaseModelFiles; + use crate::backend_init_gate::BACKEND_INIT_GATE; + use crate::model_load_params::ModelLoadParams; + use crate::phase_setup_failure_kind::PhaseSetupFailureKind; + + const NON_GGUF_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/Cargo.toml"); + + fn load_failure_kind(model_files: &PhaseModelFiles) -> Option { + let _gate = BACKEND_INIT_GATE + .lock() + .unwrap_or_else(PoisonError::into_inner); + let backend = Arc::new(LlamaBackend::init().expect("backend init must succeed")); + + model_files + .load_phase_state( + &backend, + ModelLoadParams { + n_gpu_layers: 0, + load_mode: LlamaLoadMode::Mmap, + }, + ) + .err() + .map(|setup_failure| setup_failure.kind()) + } + + #[test] + fn a_model_file_that_is_not_gguf_fails_as_a_model_load() { + assert_eq!( + load_failure_kind(&PhaseModelFiles { + model_path: PathBuf::from(NON_GGUF_PATH), + mmproj_path: None, + }), + Some(PhaseSetupFailureKind::ModelLoad) + ); + } + + #[test] + fn a_non_unicode_mmproj_path_is_refused_before_the_model_loads() { + assert_eq!( + load_failure_kind(&PhaseModelFiles { + model_path: PathBuf::from(NON_GGUF_PATH), + mmproj_path: Some(PathBuf::from(OsString::from_vec(vec![0xff]))), + }), + Some(PhaseSetupFailureKind::MmprojPathNotUnicode) + ); + } +} diff --git a/llama-cpp-test-harness/src/phase_setup.rs b/llama-cpp-test-model-provisioning/src/phase_setup.rs similarity index 91% rename from llama-cpp-test-harness/src/phase_setup.rs rename to llama-cpp-test-model-provisioning/src/phase_setup.rs index 69a6aa874..ca29caea1 100644 --- a/llama-cpp-test-harness/src/phase_setup.rs +++ b/llama-cpp-test-model-provisioning/src/phase_setup.rs @@ -1,20 +1,18 @@ use std::sync::Arc; use std::sync::OnceLock; -use anyhow::Error; -use anyhow::Result; - use llama_cpp_bindings::llama_backend::LlamaBackend; use crate::load_key::LoadKey; use crate::phase_model_files::PhaseModelFiles; +use crate::phase_setup_error::PhaseSetupError; use crate::phase_state::PhaseState; pub struct PhaseSetup { key: LoadKey, backend: Arc, - model_files: OnceLock>, - phase_state: OnceLock>, + model_files: OnceLock>, + phase_state: OnceLock>, } impl PhaseSetup { @@ -32,7 +30,7 @@ impl PhaseSetup { /// /// Returns the failure of downloading the model or the mmproj file. The outcome is cached, so /// the files are downloaded at most once per phase. - pub fn download_model_files(&self) -> Result<&PhaseModelFiles, &Error> { + pub fn download_model_files(&self) -> Result<&PhaseModelFiles, &PhaseSetupError> { self.model_files .get_or_init(|| self.key.download_model_files()) .as_ref() @@ -42,7 +40,7 @@ impl PhaseSetup { /// /// Returns the cached download failure, or the failure of loading the downloaded files into /// llama.cpp. The outcome is cached, so the model is loaded at most once per phase. - pub fn load_phase_state(&self) -> Result<&PhaseState, &Error> { + pub fn load_phase_state(&self) -> Result<&PhaseState, &PhaseSetupError> { let model_files = self.download_model_files()?; self.phase_state @@ -61,10 +59,10 @@ mod tests { use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; use super::PhaseSetup; + use crate::backend_init_gate::BACKEND_INIT_GATE; use crate::gguf_source::GgufSource; use crate::load_key::LoadKey; use crate::model_load_params::ModelLoadParams; - use crate::test_backend_gate::BACKEND_INIT_GATE; const NON_GGUF_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/Cargo.toml"); diff --git a/llama-cpp-test-model-provisioning/src/phase_setup_error.rs b/llama-cpp-test-model-provisioning/src/phase_setup_error.rs new file mode 100644 index 000000000..1705a78d1 --- /dev/null +++ b/llama-cpp-test-model-provisioning/src/phase_setup_error.rs @@ -0,0 +1,51 @@ +use std::path::PathBuf; + +use hf_hub::api::sync::ApiError; +use llama_cpp_bindings::error::llama_model_load_error::LlamaModelLoadError; +use llama_cpp_bindings::mtmd::mtmd_init_error::MtmdInitError; + +use crate::gguf_source::GgufSource; +use crate::phase_setup_failure_kind::PhaseSetupFailureKind; + +#[derive(Debug, thiserror::Error)] +pub enum PhaseSetupError { + #[error("the model {model_source} could not be downloaded: {cause}")] + ModelDownload { + model_source: GgufSource, + #[source] + cause: ApiError, + }, + #[error("the mmproj {mmproj_source} could not be downloaded: {cause}")] + MmprojDownload { + mmproj_source: GgufSource, + #[source] + cause: ApiError, + }, + #[error("the mmproj path {} is not Unicode", mmproj_path.display())] + MmprojPathNotUnicode { mmproj_path: PathBuf }, + #[error("the model {} could not be loaded: {cause}", model_path.display())] + ModelLoad { + model_path: PathBuf, + #[source] + cause: LlamaModelLoadError, + }, + #[error("the mmproj {} could not be loaded: {cause}", mmproj_path.display())] + MmprojLoad { + mmproj_path: PathBuf, + #[source] + cause: MtmdInitError, + }, +} + +impl PhaseSetupError { + #[must_use] + pub const fn kind(&self) -> PhaseSetupFailureKind { + match self { + Self::ModelDownload { .. } => PhaseSetupFailureKind::ModelDownload, + Self::MmprojDownload { .. } => PhaseSetupFailureKind::MmprojDownload, + Self::MmprojPathNotUnicode { .. } => PhaseSetupFailureKind::MmprojPathNotUnicode, + Self::ModelLoad { .. } => PhaseSetupFailureKind::ModelLoad, + Self::MmprojLoad { .. } => PhaseSetupFailureKind::MmprojLoad, + } + } +} diff --git a/llama-cpp-test-model-provisioning/src/phase_setup_failure_kind.rs b/llama-cpp-test-model-provisioning/src/phase_setup_failure_kind.rs new file mode 100644 index 000000000..729fae7cc --- /dev/null +++ b/llama-cpp-test-model-provisioning/src/phase_setup_failure_kind.rs @@ -0,0 +1,27 @@ +use std::fmt; +use std::fmt::Display; +use std::fmt::Formatter; + +/// Which step of setting up a phase failed, as a test declares the failure it expects. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PhaseSetupFailureKind { + ModelDownload, + MmprojDownload, + MmprojPathNotUnicode, + ModelLoad, + MmprojLoad, +} + +impl Display for PhaseSetupFailureKind { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + let name = match self { + Self::ModelDownload => "ModelDownload", + Self::MmprojDownload => "MmprojDownload", + Self::MmprojPathNotUnicode => "MmprojPathNotUnicode", + Self::ModelLoad => "ModelLoad", + Self::MmprojLoad => "MmprojLoad", + }; + + formatter.write_str(name) + } +} diff --git a/llama-cpp-test-harness/src/phase_state.rs b/llama-cpp-test-model-provisioning/src/phase_state.rs similarity index 100% rename from llama-cpp-test-harness/src/phase_state.rs rename to llama-cpp-test-model-provisioning/src/phase_state.rs From dededfd05db2a03c3f4c3e0ddc8001b5de606115 Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Mon, 2 Nov 2026 19:35:47 +0100 Subject: [PATCH 82/82] Name every trial after its model, mmproj and context sizes and refuse two trials with one name --- llama-cpp-test-harness-macros/src/expand.rs | 48 +++++++++++-- llama-cpp-test-harness/Cargo.toml | 4 ++ llama-cpp-test-harness/src/execution_plan.rs | 69 +++++++++++++++---- .../src/harness_run_error.rs | 5 ++ .../src/run_to_conclusions.rs | 4 +- .../tests/duplicate_trial_names.rs | 62 +++++++++++++++++ .../tests/phase_conclusions.rs | 2 +- 7 files changed, 173 insertions(+), 21 deletions(-) create mode 100644 llama-cpp-test-harness/tests/duplicate_trial_names.rs diff --git a/llama-cpp-test-harness-macros/src/expand.rs b/llama-cpp-test-harness-macros/src/expand.rs index e3b37e25f..c5f48e297 100644 --- a/llama-cpp-test-harness-macros/src/expand.rs +++ b/llama-cpp-test-harness-macros/src/expand.rs @@ -157,12 +157,26 @@ fn build_expected_outcome_literal(expected_outcome: &ParsedExpectedOutcome) -> T } } +fn trial_name(args: &ParsedArgs, signature: &Signature) -> String { + let mmproj = args + .mmproj_source + .as_ref() + .map_or_else(String::new, |mmproj_source| { + format!(" mmproj={}", mmproj_source.display_suffix()) + }); + + format!( + "{fn_name}[{model}{mmproj} n_ctx={n_ctx} n_batch={n_batch} n_ubatch={n_ubatch}]", + fn_name = signature.ident, + model = args.model_source.display_suffix(), + n_ctx = args.context_params.n_ctx, + n_batch = args.context_params.n_batch, + n_ubatch = args.context_params.n_ubatch, + ) +} + fn build_registration(args: &ParsedArgs, signature: &Signature) -> TokenStream { - let fn_name = &signature.ident; - let trial_name = format!( - "{fn_name}[{suffix}]", - suffix = args.model_source.display_suffix() - ); + let trial_name = trial_name(args, signature); let model_source_literal = build_model_source_literal(&args.model_source); let mmproj_source_literal = build_mmproj_source_literal(args.mmproj_source.as_ref()); let gpu_layers = args.model_load_params.n_gpu_layers; @@ -238,6 +252,8 @@ mod tests { use super::build_model_source_literal; use super::build_test_fn_literal; use super::expand; + use super::trial_name; + use crate::parsed_args::ParsedArgs; use crate::parsed_expected_failure::ParsedExpectedFailure; use crate::parsed_expected_outcome::ParsedExpectedOutcome; use crate::parsed_source::ParsedSource; @@ -279,7 +295,7 @@ mod tests { fn my_test(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } ::inventory::submit! { ::llama_cpp_test_harness::llama_test_registration::LlamaTestRegistration { - name: "my_test[bar.gguf]", + name: "my_test[bar.gguf n_ctx=1 n_batch=1 n_ubatch=1]", key: ::llama_cpp_test_model_provisioning::load_key::LoadKey { model_source: ::llama_cpp_test_model_provisioning::gguf_source::GgufSource::HuggingFace { repo: "foo", @@ -369,6 +385,26 @@ mod tests { ); } + #[test] + fn a_trial_with_an_mmproj_names_it_beside_its_model_and_context_sizes() { + let args: ParsedArgs = parse2(quote! { + model_source = HuggingFace("foo", "bar.gguf"), + mmproj_source = LocalPath("/models/mmproj.gguf"), + n_gpu_layers = 0, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64 + }) + .expect("the attribute must parse"); + let item_fn: ItemFn = parse2(well_formed_function()).expect("the item must be a function"); + + assert_eq!( + trial_name(&args, &item_fn.sig), + "my_test[bar.gguf mmproj=mmproj.gguf n_ctx=512 n_batch=128 n_ubatch=64]" + ); + } + #[test] fn a_function_returning_a_result_registers_as_fallible() { assert_eq!( diff --git a/llama-cpp-test-harness/Cargo.toml b/llama-cpp-test-harness/Cargo.toml index ad2ca129d..06c9b9c61 100644 --- a/llama-cpp-test-harness/Cargo.toml +++ b/llama-cpp-test-harness/Cargo.toml @@ -22,6 +22,10 @@ metal = ["llama-cpp-bindings/metal"] vulkan = ["llama-cpp-bindings/vulkan"] rocm = ["llama-cpp-bindings/rocm"] +[[test]] +name = "duplicate_trial_names" +harness = false + [[test]] name = "phase_conclusions" harness = false diff --git a/llama-cpp-test-harness/src/execution_plan.rs b/llama-cpp-test-harness/src/execution_plan.rs index 130b70452..5315c3bdb 100644 --- a/llama-cpp-test-harness/src/execution_plan.rs +++ b/llama-cpp-test-harness/src/execution_plan.rs @@ -1,4 +1,5 @@ use std::collections::BTreeMap; +use std::collections::BTreeSet; use std::sync::Arc; use libtest_mimic::Arguments; @@ -7,9 +8,26 @@ use libtest_mimic::Conclusion; use llama_cpp_bindings::llama_backend::LlamaBackend; use crate::execution_phase::ExecutionPhase; +use crate::harness_run_error::HarnessRunError; use crate::llama_test_registration::LlamaTestRegistration; use crate::scheduled_phase::ScheduledPhase; +fn refuse_duplicate_trial_names( + registrations: &[&'static LlamaTestRegistration], +) -> Result<(), HarnessRunError> { + let mut trial_names = BTreeSet::new(); + + registrations.iter().try_for_each(|registration| { + if trial_names.insert(registration.name) { + Ok(()) + } else { + Err(HarnessRunError::DuplicateTrialName { + name: registration.name, + }) + } + }) +} + fn collect_inventory_registrations() -> Vec<&'static LlamaTestRegistration> { inventory::iter:: .into_iter() @@ -21,8 +39,15 @@ pub struct ExecutionPlan { } impl ExecutionPlan { - #[must_use] - pub fn from_registrations(registrations: &[&'static LlamaTestRegistration]) -> Self { + /// # Errors + /// + /// Returns [`HarnessRunError::DuplicateTrialName`] when two registrations share a trial name, + /// since their results could not be told apart. + pub fn from_registrations( + registrations: &[&'static LlamaTestRegistration], + ) -> Result { + refuse_duplicate_trial_names(registrations)?; + let mut by_key: BTreeMap<_, Vec<&'static LlamaTestRegistration>> = BTreeMap::new(); for registration in registrations { by_key @@ -35,13 +60,14 @@ impl ExecutionPlan { registrations.sort_by_key(|registration| registration.name); phases.push(ExecutionPhase { key, registrations }); } - Self { phases } + Ok(Self { phases }) } - #[must_use] - pub fn from_inventory() -> Self { - let registrations = collect_inventory_registrations(); - Self::from_registrations(®istrations) + /// # Errors + /// + /// Returns the [`HarnessRunError`] of [`Self::from_registrations`] for every registered trial. + pub fn from_inventory() -> Result { + Self::from_registrations(&collect_inventory_registrations()) } #[must_use] @@ -166,16 +192,32 @@ mod tests { expected_outcome: ExpectedOutcome::Success, }; + #[test] + fn registrations_sharing_a_trial_name_are_refused() { + assert_eq!( + ExecutionPlan::from_registrations(&[®_BETA_A, ®_BETA_A]) + .err() + .map(|refusal| refusal.to_string()), + Some( + "two trials are named `alpha`; every trial needs its own name so its result \ + can be told apart" + .to_owned() + ) + ); + } + #[test] fn from_registrations_with_empty_input_yields_empty_plan() { - let plan = ExecutionPlan::from_registrations(&[]); + let plan = ExecutionPlan::from_registrations(&[]) + .expect("registrations with distinct names form a plan"); assert!(plan.phases.is_empty()); } #[test] fn registrations_with_same_load_key_collapse_to_one_phase() { - let plan = ExecutionPlan::from_registrations(&[®_BETA_A, ®_BETA_B]); + let plan = ExecutionPlan::from_registrations(&[®_BETA_A, ®_BETA_B]) + .expect("registrations with distinct names form a plan"); assert_eq!(plan.phases.len(), 1); assert_eq!(plan.phases[0].registrations.len(), 2); @@ -183,7 +225,8 @@ mod tests { #[test] fn registrations_with_distinct_load_keys_form_phases_in_load_key_sort_order() { - let plan = ExecutionPlan::from_registrations(&[®_BETA_A, ®_ALPHA_Z]); + let plan = ExecutionPlan::from_registrations(&[®_BETA_A, ®_ALPHA_Z]) + .expect("registrations with distinct names form a plan"); assert_eq!(plan.phases.len(), 2); assert_eq!( @@ -204,7 +247,8 @@ mod tests { #[test] fn within_a_phase_registrations_sort_by_name() { - let plan = ExecutionPlan::from_registrations(&[®_BETA_B, ®_BETA_A]); + let plan = ExecutionPlan::from_registrations(&[®_BETA_B, ®_BETA_A]) + .expect("registrations with distinct names form a plan"); assert_eq!(plan.phases.len(), 1); assert_eq!(plan.phases[0].registrations[0].name, "alpha"); @@ -213,7 +257,8 @@ mod tests { #[test] fn registrations_sharing_a_load_key_but_differing_context_params_stay_in_one_phase() { - let plan = ExecutionPlan::from_registrations(&[®_BETA_A, ®_BETA_DIFFERENT_CONTEXT]); + let plan = ExecutionPlan::from_registrations(&[®_BETA_A, ®_BETA_DIFFERENT_CONTEXT]) + .expect("registrations with distinct names form a plan"); assert_eq!(plan.phases.len(), 1); assert_eq!(plan.phases[0].registrations.len(), 2); diff --git a/llama-cpp-test-harness/src/harness_run_error.rs b/llama-cpp-test-harness/src/harness_run_error.rs index 58c5aa670..dacae6587 100644 --- a/llama-cpp-test-harness/src/harness_run_error.rs +++ b/llama-cpp-test-harness/src/harness_run_error.rs @@ -8,6 +8,11 @@ pub enum HarnessRunError { "the test harness requires --test-threads=1 (or unset); got --test-threads={requested}" )] ConflictingTestThreads { requested: usize }, + #[error( + "two trials are named `{name}`; every trial needs its own name so its result can be told \ + apart" + )] + DuplicateTrialName { name: &'static str }, #[error("failed to initialise the llama backend: {0}")] BackendInit(#[from] LlamaCppError), } diff --git a/llama-cpp-test-harness/src/run_to_conclusions.rs b/llama-cpp-test-harness/src/run_to_conclusions.rs index ca64febce..6191bf24b 100644 --- a/llama-cpp-test-harness/src/run_to_conclusions.rs +++ b/llama-cpp-test-harness/src/run_to_conclusions.rs @@ -12,12 +12,12 @@ use crate::parse_harness_arguments::parse_harness_arguments; /// # Errors /// /// Returns [`HarnessRunError`] when the CLI arguments conflict with the harness's single-thread -/// requirement or the llama backend cannot be initialised. Surfacing these as a typed error keeps +/// requirement, two registered trials share a name, or the llama backend cannot be initialised. Surfacing these as a typed error keeps /// the failure explicit instead of aborting the process with a panic. pub fn run_to_conclusions(arguments: Arguments) -> Result, HarnessRunError> { let arguments = parse_harness_arguments(arguments)?; + let plan = ExecutionPlan::from_inventory()?; let backend = Arc::new(LlamaBackend::init()?); - let plan = ExecutionPlan::from_inventory(); Ok(plan.run(&backend, &arguments)) } diff --git a/llama-cpp-test-harness/tests/duplicate_trial_names.rs b/llama-cpp-test-harness/tests/duplicate_trial_names.rs new file mode 100644 index 000000000..1bfed55f3 --- /dev/null +++ b/llama-cpp-test-harness/tests/duplicate_trial_names.rs @@ -0,0 +1,62 @@ +use std::process::ExitCode; + +use inventory::submit; +use libtest_mimic::Arguments; + +use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; +use llama_cpp_test_harness::context_params::ContextParams; +use llama_cpp_test_harness::expected_outcome::ExpectedOutcome; +use llama_cpp_test_harness::harness_run_error::HarnessRunError; +use llama_cpp_test_harness::llama_test_fn::LlamaTestFn; +use llama_cpp_test_harness::llama_test_registration::LlamaTestRegistration; +use llama_cpp_test_harness::no_op::no_op; +use llama_cpp_test_harness::run_to_conclusions::run_to_conclusions; +use llama_cpp_test_model_provisioning::gguf_source::GgufSource; +use llama_cpp_test_model_provisioning::load_key::LoadKey; +use llama_cpp_test_model_provisioning::model_load_params::ModelLoadParams; + +const NAME_REGISTERED_TWICE: &str = "a_trial_registered_twice_under_one_name"; + +const REGISTRATION: LlamaTestRegistration = LlamaTestRegistration { + name: NAME_REGISTERED_TWICE, + key: LoadKey { + model_source: GgufSource::LocalPath("Cargo.toml"), + mmproj_source: None, + model_load_params: ModelLoadParams { + n_gpu_layers: 0, + load_mode: LlamaLoadMode::Mmap, + }, + }, + context_params: ContextParams { + n_ctx: 1, + n_batch: 1, + n_ubatch: 1, + n_seq_max: 1, + n_threads_batch: None, + embeddings: false, + }, + func: LlamaTestFn::Infallible(no_op), + expected_outcome: ExpectedOutcome::Success, +}; + +submit! { REGISTRATION } + +submit! { REGISTRATION } + +fn main() -> ExitCode { + let refusal = run_to_conclusions(Arguments::from_args()).err(); + + if refusal + == Some(HarnessRunError::DuplicateTrialName { + name: NAME_REGISTERED_TWICE, + }) + { + ExitCode::SUCCESS + } else { + eprintln!( + "duplicate_trial_names: the harness did not refuse the duplicate name: {refusal:?}" + ); + + ExitCode::FAILURE + } +} diff --git a/llama-cpp-test-harness/tests/phase_conclusions.rs b/llama-cpp-test-harness/tests/phase_conclusions.rs index 014c92f29..b2b090246 100644 --- a/llama-cpp-test-harness/tests/phase_conclusions.rs +++ b/llama-cpp-test-harness/tests/phase_conclusions.rs @@ -110,7 +110,7 @@ fn phase_b_second_passing_trial(fixture: &LlamaFixture<'_>) { n_ctx = 512, n_batch = 128, n_ubatch = 64, - mmproj_source = LocalPath("/nonexistent/llama-cpp-test-harness/no-such-mmproj.gguf"), + mmproj_source = LocalPath("/nonexistent/llama-cpp-test-harness/no-such-local-mmproj.gguf"), expected_failure = SetupFailure( MmprojLoad, "the local mmproj file does not exist, so llama.cpp cannot load it"