diff --git a/Cargo.toml b/Cargo.toml index 814affd8..28af98c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,7 @@ llama-cpp-log-decoder = { path = "llama-cpp-log-decoder", version = "=0.27.0" } llama-cpp-test-harness = { path = "llama-cpp-test-harness", version = "=0.27.0" } llama-cpp-test-harness-macros = { path = "llama-cpp-test-harness-macros", version = "=0.27.0" } llama-cpp-test-model-provisioning = { path = "llama-cpp-test-model-provisioning", version = "=0.27.0" } -llama-cpp-fault-injection = { path = "llama-cpp-fault-injection" } +llama-cpp-fault-injection = { path = "llama-cpp-fault-injection", version = "=0.27.0" } llama-cpp-wrapper-error-fixture = { path = "llama-cpp-wrapper-error-fixture" } llama-cpp-wrapper-sources = { path = "llama-cpp-wrapper-sources", version = "=0.27.0" } llguidance = "=1.7.0" diff --git a/llama-cpp-bindings-sys/wrapper_chat_session.cpp b/llama-cpp-bindings-sys/wrapper_chat_session.cpp index abe55c4d..4f9e1c47 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_session.cpp +++ b/llama-cpp-bindings-sys/wrapper_chat_session.cpp @@ -29,7 +29,7 @@ #include struct llama_rs_chat_session { - common_chat_session session; + std::shared_ptr session; common_chat_input generation_prompt; std::vector generation_prompt_tokens; common_params_sampling sampling; @@ -37,9 +37,16 @@ struct llama_rs_chat_session { std::unordered_set trigger_rules; }; -struct llama_rs_chat_spans { +struct llama_rs_chat_reading { + std::shared_ptr session; + bool is_partial = false; + size_t generation_prompt_length = 0; + common_peg_parse_context context; + common_peg_parse_result result; + bool matches_format = false; size_t parsed_length = 0; std::vector spans; + std::string format_name; }; namespace { @@ -138,8 +145,36 @@ auto chat_input_of( return input; } -auto parse_flags_of(bool is_partial) -> common_peg_parse_flags { - return is_partial ? COMMON_PEG_PARSE_FLAG_LENIENT : COMMON_PEG_PARSE_FLAG_COMPLETE; +auto finished_output_of(const llama_rs_chat_reading & reading) -> common_chat_input { + common_chat_input output; + output.text = reading.context.input.substr(reading.generation_prompt_length); + output.tokens.assign( + reading.context.tokens.begin() + static_cast(reading.generation_prompt_length), + reading.context.tokens.end()); + + return output; +} + +auto finished_output_parser_params_of(const llama_rs_chat_reading & reading) -> common_chat_parser_params { + common_chat_parser_params parser_params; + parser_params.format = reading.session->format(); + parser_params.generation_prompt.text = reading.context.input.substr(0, reading.generation_prompt_length); + parser_params.generation_prompt.tokens.assign( + reading.context.tokens.begin(), + reading.context.tokens.begin() + static_cast(reading.generation_prompt_length)); + + return parser_params; +} + +auto mapper_of(common_chat_format format, common_chat_msg & message) -> std::unique_ptr { + if (format == COMMON_CHAT_FORMAT_PEG_GEMMA4) { + return std::make_unique(message); + } + if (format == COMMON_CHAT_FORMAT_PEG_MINIMAX_M3) { + return std::make_unique(message); + } + + return std::make_unique(message); } } // namespace @@ -147,7 +182,9 @@ extern "C" auto llama_rs_chat_session_create( llama_rs_chat_templates_handle templates, const struct llama_vocab * vocab, const char * messages_json, + size_t messages_json_length, const char * tools_json, + size_t tools_json_length, llama_rs_chat_session_options options, llama_rs_chat_session_handle * out_session, char ** out_error) -> llama_rs_chat_session_create_status { @@ -179,19 +216,11 @@ extern "C" auto llama_rs_chat_session_create( try { common_chat_templates_inputs inputs; - inputs.messages = common_chat_msgs_parse_oaicompat(common_json::parse(messages_json)); + inputs.messages = common_chat_msgs_parse_oaicompat( + common_json::parse(std::string(messages_json, messages_json_length))); - try { - inputs.tools = common_chat_tools_parse_oaicompat(common_json::parse(tools_json)); - } catch (const common_json_error & err) { - return llama_rs_report_exception_message( - out_error, err.what(), LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_NOT_JSON, - LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED); - } catch (const std::runtime_error & err) { - return llama_rs_report_exception_message( - out_error, err.what(), LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_INVALID, - LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED); - } + inputs.tools = common_chat_tools_parse_oaicompat( + common_json::parse(std::string(tools_json, tools_json_length))); inputs.add_generation_prompt = options.assistant_turn == LLAMA_RS_CHAT_ASSISTANT_TURN_NEW; inputs.continue_final_message = continuation_of(options.assistant_turn); @@ -203,7 +232,7 @@ extern "C" auto llama_rs_chat_session_create( auto chat_session = std::make_unique(); try { - chat_session->session = common_chat_session(templates->templates.get(), vocab, inputs); + chat_session->session = std::make_shared(templates->templates.get(), vocab, inputs); } catch (const std::bad_alloc &) { throw; } catch (const std::exception & err) { @@ -212,16 +241,16 @@ extern "C" auto llama_rs_chat_session_create( LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED); } - chat_session->generation_prompt = common_chat_input_tokenize(vocab, chat_session->session.generation_prompt()); + chat_session->generation_prompt = common_chat_input_tokenize(vocab, chat_session->session->generation_prompt()); for (const llama_token token : chat_session->generation_prompt.tokens) { if (token != LLAMA_TOKEN_NULL) { chat_session->generation_prompt_tokens.push_back(token); } } - chat_session->session.apply_sampling(chat_session->sampling); + chat_session->session->apply_sampling(chat_session->sampling); chat_session->preserved_tokens.assign( chat_session->sampling.preserved_tokens.begin(), chat_session->sampling.preserved_tokens.end()); - chat_session->trigger_rules = trigger_rules_of(chat_session->session.parser()); + chat_session->trigger_rules = trigger_rules_of(chat_session->session->parser()); *out_session = chat_session.release(); @@ -240,7 +269,7 @@ extern "C" auto llama_rs_chat_session_free(llama_rs_chat_session_handle session) } extern "C" auto llama_rs_chat_session_prompt_part_count(llama_rs_chat_session_handle session) -> size_t { - return session->session.prompt().parts.size(); + return session->session->prompt().parts.size(); } extern "C" auto llama_rs_chat_session_prompt_part_text( @@ -249,19 +278,19 @@ extern "C" auto llama_rs_chat_session_prompt_part_text( size_t * out_length) -> const char * { return llama_rs_text_view_unless_fault_injected( LLAMA_RS_FAULT_POINT_CHAT_SESSION_PROMPT_PART_TEXT_NOT_UTF8, - session->session.prompt().parts.at(index).text, + session->session->prompt().parts.at(index).text, out_length); } extern "C" auto llama_rs_chat_session_prompt_part_is_input(llama_rs_chat_session_handle session, size_t index) -> bool { - return session->session.prompt().parts.at(index).is_input; + return session->session->prompt().parts.at(index).is_input; } extern "C" auto llama_rs_chat_session_grammar(llama_rs_chat_session_handle session, size_t * out_length) -> const char * { return llama_rs_text_view_unless_fault_injected( - LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_NOT_UTF8, session->session.grammar(), out_length); + LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_NOT_UTF8, session->session->grammar(), out_length); } extern "C" auto llama_rs_chat_session_grammar_lazy(llama_rs_chat_session_handle session) -> bool { @@ -318,7 +347,7 @@ extern "C" auto llama_rs_chat_session_preserved_token(llama_rs_chat_session_hand extern "C" auto llama_rs_chat_session_generation_prompt(llama_rs_chat_session_handle session, size_t * out_length) -> const char * { - return llama_rs_string_view(session->session.generation_prompt(), out_length); + return llama_rs_string_view(session->session->generation_prompt(), out_length); } extern "C" auto llama_rs_chat_session_generation_prompt_token_count(llama_rs_chat_session_handle session) -> size_t { @@ -332,162 +361,164 @@ extern "C" auto llama_rs_chat_session_generation_prompt_token(llama_rs_chat_sess extern "C" auto llama_rs_chat_session_thinking_start_tag(llama_rs_chat_session_handle session, size_t * out_length) -> const char * { - return llama_rs_string_view(session->session.thinking_start_tag(), out_length); + return llama_rs_string_view(session->session->thinking_start_tag(), out_length); } extern "C" auto llama_rs_chat_session_thinking_end_tag_count(llama_rs_chat_session_handle session) -> size_t { - return session->session.thinking_end_tags().size(); + return session->session->thinking_end_tags().size(); } extern "C" auto llama_rs_chat_session_thinking_end_tag( llama_rs_chat_session_handle session, size_t index, size_t * out_length) -> const char * { - return llama_rs_string_view(session->session.thinking_end_tags().at(index), out_length); + return llama_rs_string_view(session->session->thinking_end_tags().at(index), out_length); } -extern "C" auto llama_rs_chat_session_parse( +extern "C" auto llama_rs_chat_session_read( llama_rs_chat_session_handle session, const char * text, size_t text_length, const llama_token * byte_tokens, bool is_partial, - llama_rs_parsed_chat_handle * out_handle, - char ** out_error) -> llama_rs_chat_session_parse_status { - if (out_handle != nullptr) { - *out_handle = nullptr; + llama_rs_chat_reading_handle * out_reading, + char ** out_error) -> llama_rs_chat_session_read_status { + if (out_reading != nullptr) { + *out_reading = nullptr; } if (out_error != nullptr) { *out_error = nullptr; } if (session == nullptr) { - return LLAMA_RS_CHAT_SESSION_PARSE_NULL_SESSION_ARG; + return LLAMA_RS_CHAT_SESSION_READ_NULL_SESSION_ARG; } if (text == nullptr) { - return LLAMA_RS_CHAT_SESSION_PARSE_NULL_TEXT_ARG; + return LLAMA_RS_CHAT_SESSION_READ_NULL_TEXT_ARG; } if (byte_tokens == nullptr) { - return LLAMA_RS_CHAT_SESSION_PARSE_NULL_BYTE_TOKENS_ARG; + return LLAMA_RS_CHAT_SESSION_READ_NULL_BYTE_TOKENS_ARG; } - if (out_handle == nullptr) { - return LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_HANDLE_ARG; + if (out_reading == nullptr) { + return LLAMA_RS_CHAT_SESSION_READ_NULL_OUT_READING_ARG; } if (out_error == nullptr) { - return LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_ERROR_ARG; + return LLAMA_RS_CHAT_SESSION_READ_NULL_OUT_ERROR_ARG; } try { - llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_CHAT_SESSION_PARSE_THROWS); + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_CHAT_SESSION_READ_THROWS); - common_chat_parser_params parser_params; - parser_params.format = session->session.format(); - parser_params.generation_prompt = session->generation_prompt; + common_chat_input parse_input = chat_input_of(session->generation_prompt, text, text_length, byte_tokens); + const size_t generation_prompt_length = session->generation_prompt.size(); + auto reading = std::make_unique(); - auto parsed = std::make_unique(); + reading->session = session->session; + reading->is_partial = is_partial; + reading->generation_prompt_length = generation_prompt_length; + reading->context.input = std::move(parse_input.text); + reading->context.tokens = std::move(parse_input.tokens); + reading->context.flags = is_partial ? COMMON_PEG_PARSE_FLAG_LENIENT : COMMON_PEG_PARSE_FLAG_COMPLETE; - try { - parsed->message = common_chat_peg_parse( - session->session.parser(), - chat_input_of(common_chat_input(), text, text_length, byte_tokens), - is_partial, - parser_params); - } catch (const std::bad_alloc &) { - throw; - } catch (const std::runtime_error & err) { - return llama_rs_report_exception_message( - out_error, err.what(), LLAMA_RS_CHAT_SESSION_PARSE_OUTPUT_DOES_NOT_MATCH_FORMAT, - LLAMA_RS_CHAT_SESSION_PARSE_ERROR_STRING_ALLOCATION_FAILED); + reading->result = session->session->parser().parse(reading->context); + reading->matches_format = is_partial + ? !reading->result.fail() || reading->result.end > 0 + : !reading->result.fail() && reading->result.end == reading->context.input.size(); + + if (!reading->matches_format) { + reading->format_name = common_chat_format_name(session->session->format()); + *out_reading = reading.release(); + + return LLAMA_RS_CHAT_SESSION_READ_OK; } - *out_handle = parsed.release(); + reading->parsed_length = + reading->result.end > generation_prompt_length ? reading->result.end - generation_prompt_length : 0; + + reading->context.ast.visit(reading->result, [&](const common_peg_ast_node & node) -> void { + const auto kind = span_kind_of_node(session->trigger_rules, node); + if (!kind || node.end <= generation_prompt_length) { + return; + } + reading->spans.push_back({ + *kind, + node.start > generation_prompt_length ? node.start - generation_prompt_length : 0, + node.end - generation_prompt_length, + }); + }); + + *out_reading = reading.release(); - return LLAMA_RS_CHAT_SESSION_PARSE_OK; + return LLAMA_RS_CHAT_SESSION_READ_OK; } catch (...) { return llama_rs_report_current_exception( out_error, - LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_CHAT_SESSION_PARSE_ERROR_STRING_ALLOCATION_FAILED); + LLAMA_RS_CHAT_SESSION_READ_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_SESSION_READ_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_CHAT_SESSION_READ_ERROR_STRING_ALLOCATION_FAILED); } } -extern "C" auto llama_rs_chat_session_classify( - llama_rs_chat_session_handle session, - const char * text, - size_t text_length, - const llama_token * byte_tokens, - bool is_partial, - llama_rs_chat_spans_handle * out_spans, - char ** out_error) -> llama_rs_chat_session_classify_status { - if (out_spans != nullptr) { - *out_spans = nullptr; +extern "C" auto llama_rs_chat_reading_matches_format(llama_rs_chat_reading_handle reading) -> bool { + return reading->matches_format; +} + +extern "C" auto llama_rs_chat_reading_spans_view(llama_rs_chat_reading_handle reading) -> llama_rs_chat_spans_view { + return {reading->spans.data(), reading->spans.size(), reading->parsed_length}; +} + +extern "C" auto llama_rs_chat_reading_format_name(llama_rs_chat_reading_handle reading, size_t * out_length) + -> const char * { + return llama_rs_text_view_unless_fault_injected( + LLAMA_RS_FAULT_POINT_CHAT_READING_FORMAT_NAME_NOT_UTF8, reading->format_name, out_length); +} + +extern "C" auto llama_rs_chat_reading_parse( + llama_rs_chat_reading_handle reading, + llama_rs_parsed_chat_handle * out_handle, + char ** out_error) -> llama_rs_chat_reading_parse_status { + if (out_handle != nullptr) { + *out_handle = nullptr; } if (out_error != nullptr) { *out_error = nullptr; } - if (session == nullptr) { - return LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_SESSION_ARG; - } - if (text == nullptr) { - return LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_TEXT_ARG; - } - if (byte_tokens == nullptr) { - return LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_BYTE_TOKENS_ARG; + if (reading == nullptr) { + return LLAMA_RS_CHAT_READING_PARSE_NULL_READING_ARG; } - if (out_spans == nullptr) { - return LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_SPANS_ARG; + if (out_handle == nullptr) { + return LLAMA_RS_CHAT_READING_PARSE_NULL_OUT_HANDLE_ARG; } if (out_error == nullptr) { - return LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_ERROR_ARG; + return LLAMA_RS_CHAT_READING_PARSE_NULL_OUT_ERROR_ARG; } try { - llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_CHAT_SESSION_CLASSIFY_THROWS); + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_CHAT_READING_PARSE_THROWS); - common_chat_input parse_input = chat_input_of(session->generation_prompt, text, text_length, byte_tokens); - const size_t generation_prompt_length = session->generation_prompt.size(); - - common_peg_parse_context context( - std::move(parse_input.text), std::move(parse_input.tokens), parse_flags_of(is_partial)); - const common_peg_parse_result result = session->session.parser().parse(context); + auto parsed = std::make_unique(); + parsed->message.role = "assistant"; - if (result.fail() && (!is_partial || result.end == 0)) { - return LLAMA_RS_CHAT_SESSION_CLASSIFY_OUTPUT_DOES_NOT_MATCH_FORMAT; + if (reading->is_partial) { + mapper_of(reading->session->format(), parsed->message)->from_ast(reading->context.ast, reading->result); + } else { + parsed->message = common_chat_peg_parse( + reading->session->parser(), finished_output_of(*reading), false, finished_output_parser_params_of(*reading)); } - auto spans = std::make_unique(); - spans->parsed_length = result.end > generation_prompt_length ? result.end - generation_prompt_length : 0; - - context.ast.visit(result, [&](const common_peg_ast_node & node) -> void { - const auto kind = span_kind_of_node(session->trigger_rules, node); - if (!kind || node.end <= generation_prompt_length) { - return; - } - spans->spans.push_back({ - *kind, - node.start > generation_prompt_length ? node.start - generation_prompt_length : 0, - node.end - generation_prompt_length, - }); - }); - - *out_spans = spans.release(); + *out_handle = parsed.release(); - return LLAMA_RS_CHAT_SESSION_CLASSIFY_OK; + return LLAMA_RS_CHAT_READING_PARSE_OK; } catch (...) { return llama_rs_report_current_exception( out_error, - LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_CHAT_SESSION_CLASSIFY_ERROR_STRING_ALLOCATION_FAILED); + LLAMA_RS_CHAT_READING_PARSE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_READING_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_CHAT_READING_PARSE_ERROR_STRING_ALLOCATION_FAILED); } } -extern "C" auto llama_rs_chat_spans_view_of(llama_rs_chat_spans_handle spans) -> llama_rs_chat_spans_view { - return {spans->spans.data(), spans->spans.size(), spans->parsed_length}; -} - -extern "C" auto llama_rs_chat_spans_free(llama_rs_chat_spans_handle spans) -> void { - std::unique_ptr(spans).reset(); +extern "C" auto llama_rs_chat_reading_free(llama_rs_chat_reading_handle reading) -> void { + std::unique_ptr(reading).reset(); } extern "C" auto llama_rs_regex_escape( diff --git a/llama-cpp-bindings-sys/wrapper_chat_session.h b/llama-cpp-bindings-sys/wrapper_chat_session.h index 59550e8d..df4b5b49 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_session.h +++ b/llama-cpp-bindings-sys/wrapper_chat_session.h @@ -15,8 +15,8 @@ extern "C" { struct llama_rs_chat_session; typedef struct llama_rs_chat_session * llama_rs_chat_session_handle; -struct llama_rs_chat_spans; -typedef struct llama_rs_chat_spans * llama_rs_chat_spans_handle; +struct llama_rs_chat_reading; +typedef struct llama_rs_chat_reading * llama_rs_chat_reading_handle; typedef enum llama_rs_chat_tool_choice { LLAMA_RS_CHAT_TOOL_CHOICE_AUTO = 0, @@ -46,8 +46,6 @@ typedef enum llama_rs_chat_session_create_status { LLAMA_RS_CHAT_SESSION_CREATE_NULL_TOOLS_JSON_ARG, LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_SESSION_ARG, LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_ERROR_ARG, - LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_NOT_JSON, - LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_INVALID, LLAMA_RS_CHAT_SESSION_CREATE_TEMPLATE_CANNOT_SERVE_REQUEST, LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED, LLAMA_RS_CHAT_SESSION_CREATE_LLAMA_CPP_OUT_OF_MEMORY, @@ -58,7 +56,9 @@ llama_rs_chat_session_create_status llama_rs_chat_session_create( llama_rs_chat_templates_handle templates, const struct llama_vocab * vocab, const char * messages_json, + size_t messages_json_length, const char * tools_json, + size_t tools_json_length, llama_rs_chat_session_options options, llama_rs_chat_session_handle * out_session, char ** out_error); @@ -118,28 +118,6 @@ const char * llama_rs_chat_session_thinking_end_tag( size_t index, size_t * out_length); -typedef enum llama_rs_chat_session_parse_status { - LLAMA_RS_CHAT_SESSION_PARSE_OK = 0, - LLAMA_RS_CHAT_SESSION_PARSE_NULL_SESSION_ARG, - LLAMA_RS_CHAT_SESSION_PARSE_NULL_TEXT_ARG, - LLAMA_RS_CHAT_SESSION_PARSE_NULL_BYTE_TOKENS_ARG, - LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_HANDLE_ARG, - LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_ERROR_ARG, - LLAMA_RS_CHAT_SESSION_PARSE_OUTPUT_DOES_NOT_MATCH_FORMAT, - LLAMA_RS_CHAT_SESSION_PARSE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_chat_session_parse_status; - -llama_rs_chat_session_parse_status llama_rs_chat_session_parse( - llama_rs_chat_session_handle session, - const char * text, - size_t text_length, - const llama_token * byte_tokens, - bool is_partial, - llama_rs_parsed_chat_handle * out_handle, - char ** out_error); - typedef enum llama_rs_chat_span_kind { LLAMA_RS_CHAT_SPAN_KIND_REASONING_BLOCK = 0, LLAMA_RS_CHAT_SPAN_KIND_REASONING, @@ -160,31 +138,49 @@ typedef struct llama_rs_chat_spans_view { size_t parsed_length; } llama_rs_chat_spans_view; -typedef enum llama_rs_chat_session_classify_status { - LLAMA_RS_CHAT_SESSION_CLASSIFY_OK = 0, - LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_SESSION_ARG, - LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_TEXT_ARG, - LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_BYTE_TOKENS_ARG, - LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_SPANS_ARG, - LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_ERROR_ARG, - LLAMA_RS_CHAT_SESSION_CLASSIFY_OUTPUT_DOES_NOT_MATCH_FORMAT, - LLAMA_RS_CHAT_SESSION_CLASSIFY_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_chat_session_classify_status; - -llama_rs_chat_session_classify_status llama_rs_chat_session_classify( +typedef enum llama_rs_chat_session_read_status { + LLAMA_RS_CHAT_SESSION_READ_OK = 0, + LLAMA_RS_CHAT_SESSION_READ_NULL_SESSION_ARG, + LLAMA_RS_CHAT_SESSION_READ_NULL_TEXT_ARG, + LLAMA_RS_CHAT_SESSION_READ_NULL_BYTE_TOKENS_ARG, + LLAMA_RS_CHAT_SESSION_READ_NULL_OUT_READING_ARG, + LLAMA_RS_CHAT_SESSION_READ_NULL_OUT_ERROR_ARG, + LLAMA_RS_CHAT_SESSION_READ_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_CHAT_SESSION_READ_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_SESSION_READ_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_chat_session_read_status; + +llama_rs_chat_session_read_status llama_rs_chat_session_read( llama_rs_chat_session_handle session, const char * text, size_t text_length, const llama_token * byte_tokens, bool is_partial, - llama_rs_chat_spans_handle * out_spans, + llama_rs_chat_reading_handle * out_reading, char ** out_error); -llama_rs_chat_spans_view llama_rs_chat_spans_view_of(llama_rs_chat_spans_handle spans); +bool llama_rs_chat_reading_matches_format(llama_rs_chat_reading_handle reading); + +llama_rs_chat_spans_view llama_rs_chat_reading_spans_view(llama_rs_chat_reading_handle reading); + +const char * llama_rs_chat_reading_format_name(llama_rs_chat_reading_handle reading, size_t * out_length); + +typedef enum llama_rs_chat_reading_parse_status { + LLAMA_RS_CHAT_READING_PARSE_OK = 0, + LLAMA_RS_CHAT_READING_PARSE_NULL_READING_ARG, + LLAMA_RS_CHAT_READING_PARSE_NULL_OUT_HANDLE_ARG, + LLAMA_RS_CHAT_READING_PARSE_NULL_OUT_ERROR_ARG, + LLAMA_RS_CHAT_READING_PARSE_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_CHAT_READING_PARSE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_READING_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_chat_reading_parse_status; + +llama_rs_chat_reading_parse_status llama_rs_chat_reading_parse( + llama_rs_chat_reading_handle reading, + llama_rs_parsed_chat_handle * out_handle, + char ** out_error); -void llama_rs_chat_spans_free(llama_rs_chat_spans_handle spans); +void llama_rs_chat_reading_free(llama_rs_chat_reading_handle reading); typedef enum llama_rs_regex_escape_status { LLAMA_RS_REGEX_ESCAPE_OK = 0, diff --git a/llama-cpp-bindings-sys/wrapper_chat_templates.cpp b/llama-cpp-bindings-sys/wrapper_chat_templates.cpp index 4aa9f698..2f56f69d 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_templates.cpp +++ b/llama-cpp-bindings-sys/wrapper_chat_templates.cpp @@ -5,9 +5,34 @@ #include #include +#include #include +#include #include +namespace { +auto default_template_source_of(const struct llama_model * model, const char * template_override) -> std::string { + if (*template_override != '\0') { + return template_override; + } + + return llama_model_chat_template(model, nullptr); +} + +auto templates_reporting_the_parse_error(const struct llama_model * model, const char * template_override) + -> common_chat_templates_ptr { + try { + return common_chat_templates_init(model, std::string(template_override)); + } catch (const std::bad_alloc &) { + throw; + } catch (const std::exception &) { + const auto reparsed_template = std::make_unique( + default_template_source_of(model, template_override), std::string(), std::string()); + throw; + } +} +} // namespace + extern "C" auto llama_rs_chat_templates_init( const struct llama_model * model, const char * template_override, @@ -37,7 +62,7 @@ extern "C" auto llama_rs_chat_templates_init( try { auto templates = std::make_unique(); - templates->templates = common_chat_templates_init(model, std::string(template_override)); + templates->templates = templates_reporting_the_parse_error(model, template_override); *out_templates = templates.release(); diff --git a/llama-cpp-bindings-sys/wrapper_fault_injection.h b/llama-cpp-bindings-sys/wrapper_fault_injection.h index e8c07424..1b442362 100644 --- a/llama-cpp-bindings-sys/wrapper_fault_injection.h +++ b/llama-cpp-bindings-sys/wrapper_fault_injection.h @@ -13,12 +13,13 @@ typedef enum llama_rs_fault_point { LLAMA_RS_FAULT_POINT_BACKEND_DEVICE_TYPE_UNKNOWN, LLAMA_RS_FAULT_POINT_BATCH_EXT_ASSIGN_ENTRIES_THROWS, LLAMA_RS_FAULT_POINT_BATCH_EXT_INIT_THROWS, - LLAMA_RS_FAULT_POINT_CHAT_SESSION_CLASSIFY_THROWS, + LLAMA_RS_FAULT_POINT_CHAT_READING_FORMAT_NAME_NOT_UTF8, + LLAMA_RS_FAULT_POINT_CHAT_READING_PARSE_THROWS, LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_NOT_UTF8, LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_TRIGGER_TYPE_UNRECOGNIZED, LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_TRIGGER_VALUE_NOT_UTF8, - LLAMA_RS_FAULT_POINT_CHAT_SESSION_PARSE_THROWS, LLAMA_RS_FAULT_POINT_CHAT_SESSION_PROMPT_PART_TEXT_NOT_UTF8, + LLAMA_RS_FAULT_POINT_CHAT_SESSION_READ_THROWS, LLAMA_RS_FAULT_POINT_GET_LOGITS_ITH_RETURNS_NULL, LLAMA_RS_FAULT_POINT_LLGUIDANCE_FACTORY_FAILS, LLAMA_RS_FAULT_POINT_MTMD_EVAL_CHUNK_SINGLE_THROWS, diff --git a/llama-cpp-bindings-tests/src/chat_answer.rs b/llama-cpp-bindings-tests/src/chat_answer.rs index 6cce95f0..079171aa 100644 --- a/llama-cpp-bindings-tests/src/chat_answer.rs +++ b/llama-cpp-bindings-tests/src/chat_answer.rs @@ -1,39 +1,32 @@ -use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; -use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_message_parse_outcome::ChatMessageParseOutcome; use llama_cpp_bindings_types::token_usage::TokenUsage; use crate::classify_sample_loop_outcome::ClassifySampleLoopOutcome; -fn thinking_tags(session: &ChatSession<'_>) -> Vec { - let mut tags = session - .thinking_end_tags() - .expect("the session's thinking end tags must be UTF-8"); - - tags.push( - session - .thinking_start_tag() - .expect("the session's thinking start tag must be UTF-8"), - ); - - tags -} - -/// A generated answer to a chat session, with what the classifier attributed to each section. +/// A generated answer to a chat session, with what the classifier attributed to each section, +/// the session's thinking tags, and the classifier's own final parse of the answer. #[derive(Debug)] pub struct ChatAnswer { pub outcome: ClassifySampleLoopOutcome, pub usage: TokenUsage, + pub thinking_tags: Vec, + pub parse: ChatMessageParseOutcome, } impl ChatAnswer { - /// Asserts that every generated token is content, and that no thinking tag of `session` + /// Asserts that every generated token is content, and that no thinking tag of the session /// reached the content stream. /// /// # Panics /// /// Panics when the answer reasons, holds a token of no section, or leaks a thinking tag. - pub fn assert_content_without_reasoning(&self, session: &ChatSession<'_>) { - let Self { outcome, usage } = self; + pub fn assert_content_without_reasoning(&self) { + let Self { + outcome, + usage, + thinking_tags, + .. + } = self; assert_ne!(outcome.generated.text(), "", "{outcome:?}"); assert!(outcome.observed_content > 0, "{outcome:?}"); @@ -47,16 +40,16 @@ impl ChatAnswer { "{usage:?}" ); - for tag in thinking_tags(session) { + for tag in thinking_tags { assert!( - !outcome.content_stream.contains(&tag), + !outcome.content_stream.contains(tag), "the content stream leaked the thinking tag {tag:?}; {outcome:?}" ); } } /// Asserts that the answer reasons, and that the reasoning and content streams are exactly - /// what the session's parser extracts, apart from `prompted_reasoning`, the reasoning text + /// what the classifier's own final parse extracts, apart from `prompted_reasoning`, the reasoning text /// the generation prompt itself holds, which the parser shows in front of the first /// generated text. /// @@ -65,15 +58,15 @@ impl ChatAnswer { /// Panics when the parser fails on or does not recognize the answer, or when the answer /// does not reason, holds a token of no section, leaks a thinking tag, or streams text the /// parser does not extract. - pub fn assert_reasoning_matches_parse( - &self, - session: &ChatSession<'_>, - prompted_reasoning: &str, - ) { - let Self { outcome, usage } = self; - let parsed = session - .parse(&outcome.generated, ChatOutputCompleteness::Finished) - .expect("the session's parser must parse the generated answer") + pub fn assert_reasoning_matches_parse(&self, prompted_reasoning: &str) { + let Self { + outcome, + usage, + thinking_tags, + parse, + } = self; + let parsed = parse + .clone() .into_recognized() .expect("the session's parser must recognize the generated answer"); @@ -96,9 +89,9 @@ impl ChatAnswer { ); assert_eq!(outcome.content_stream, parsed.content); - for tag in thinking_tags(session) { - assert!(!outcome.reasoning_stream.contains(&tag), "{outcome:?}"); - assert!(!outcome.content_stream.contains(&tag), "{outcome:?}"); + for tag in thinking_tags { + assert!(!outcome.reasoning_stream.contains(tag), "{outcome:?}"); + assert!(!outcome.content_stream.contains(tag), "{outcome:?}"); } } } diff --git a/llama-cpp-bindings-tests/src/chat_answer_generation.rs b/llama-cpp-bindings-tests/src/chat_answer_generation.rs index d5402ca0..5e30b1bc 100644 --- a/llama-cpp-bindings-tests/src/chat_answer_generation.rs +++ b/llama-cpp-bindings-tests/src/chat_answer_generation.rs @@ -8,10 +8,24 @@ use llama_cpp_test_harness::llama_fixture::LlamaFixture; use crate::chat_answer::ChatAnswer; use crate::classify_sample_loop::ClassifySampleLoop; +fn thinking_tags(session: &ChatSession) -> Vec { + let mut tags = session + .thinking_end_tags() + .expect("the session's thinking end tags must be UTF-8"); + + tags.push( + session + .thinking_start_tag() + .expect("the session's thinking start tag must be UTF-8"), + ); + + tags +} + /// Generates the answer to a chat session from its tokenized prompt, classifying every token. pub struct ChatAnswerGeneration<'borrow, 'model> { pub fixture: &'borrow LlamaFixture<'model>, - pub session: &'borrow ChatSession<'model>, + pub session: ChatSession, pub sampler: &'borrow mut LlamaSampler, pub max_generated_tokens: i32, } @@ -19,8 +33,8 @@ pub struct ChatAnswerGeneration<'borrow, 'model> { impl ChatAnswerGeneration<'_, '_> { /// # Panics /// - /// Panics when creating the context, tokenizing or decoding the prompt, or generating the - /// answer fails. + /// Panics when creating the context, tokenizing or decoding the prompt, generating the + /// answer, or parsing the classifier's final reading fails. #[must_use] pub fn run(self) -> ChatAnswer { let Self { @@ -33,9 +47,11 @@ impl ChatAnswerGeneration<'_, '_> { .build_context() .expect("the fixture's context must build"); let prompt_tokens = session - .tokenize_prompt() + .tokenize_prompt(fixture.model) .expect("the session's prompt must tokenize"); - let mut classifier = SampledTokenClassifier::for_chat_session(session); + let thinking_tags = thinking_tags(&session); + let mut classifier = SampledTokenClassifier::for_chat_session(fixture.model, session) + .expect("the session must read its empty answer"); let mut batch = LlamaBatch::new(ContextLimits::of(&context)); classifier @@ -57,9 +73,18 @@ impl ChatAnswerGeneration<'_, '_> { } .run(); + let parse = classifier + .latest_reading() + .expect("a generated answer must have been read") + .parse() + .cloned() + .expect("the classifier's final reading must parse"); + ChatAnswer { outcome, usage: classifier.into_usage(), + thinking_tags, + parse, } } } diff --git a/llama-cpp-bindings-tests/src/classify_sample_loop.rs b/llama-cpp-bindings-tests/src/classify_sample_loop.rs index 16805f07..87e72570 100644 --- a/llama-cpp-bindings-tests/src/classify_sample_loop.rs +++ b/llama-cpp-bindings-tests/src/classify_sample_loop.rs @@ -7,8 +7,8 @@ use llama_cpp_bindings::sampling::LlamaSampler; use crate::classify_sample_loop_outcome::ClassifySampleLoopOutcome; -pub struct ClassifySampleLoop<'borrow, 'session, 'model> { - pub classifier: &'borrow mut SampledTokenClassifier<'session, 'model>, +pub struct ClassifySampleLoop<'borrow, 'model> { + pub classifier: &'borrow mut SampledTokenClassifier<'model>, pub sampler: &'borrow mut LlamaSampler, pub context: &'borrow mut LlamaContext<'model>, pub batch: &'borrow mut LlamaBatch, @@ -16,9 +16,10 @@ pub struct ClassifySampleLoop<'borrow, 'session, 'model> { pub max_generated_tokens: i32, } -impl ClassifySampleLoop<'_, '_, '_> { +impl ClassifySampleLoop<'_, '_> { /// Samples and classifies tokens until the end of generation or until - /// `max_generated_tokens` are generated. + /// `max_generated_tokens` are generated, settling the pending tokens of a generation the + /// limit stops; an end-of-generation token settles them itself. /// /// # Panics /// @@ -58,9 +59,11 @@ impl ClassifySampleLoop<'_, '_, '_> { .expect("decoding the sampled token must succeed"); } - self.classifier - .finish(&mut ingest_outcomes) - .expect("finishing the classification must succeed"); + if !outcome.eog_seen { + self.classifier + .finish(&mut ingest_outcomes) + .expect("finishing the classification must succeed"); + } for ingest_outcome in &ingest_outcomes { outcome.record_outcome(ingest_outcome); diff --git a/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs b/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs index 941a5981..a3784308 100644 --- a/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs +++ b/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs @@ -4,7 +4,9 @@ use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings::chat_session_options::ChatSessionOptions; use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; +use llama_cpp_bindings_types::chat_tool::ChatTool; + +use crate::user_message::user_message; const PREFILLED_REASONING: &str = "\nThe question is simple, so I answer it directly.\n"; const PREFILLED_CONTENT: &str = "\n\n"; @@ -16,23 +18,20 @@ const PREFILLED_CONTENT: &str = "\n\n"; /// /// # Panics /// -/// Panics when the question holds a NUL byte or the model's template cannot serve the request. +/// Panics when the model's template cannot serve the request. #[must_use] -pub fn closed_reasoning_chat_session<'model>( - model: &'model LlamaModel, +pub fn closed_reasoning_chat_session( + model: &LlamaModel, question: &str, - tools_json: &str, -) -> ChatSession<'model> { + tools: &[ChatTool], +) -> ChatSession { ChatSession::new( model, model .chat_templates() .expect("the model must have chat templates"), - &[ - LlamaChatMessage::new("user".to_owned(), question.to_owned()) - .expect("the question must hold no NUL byte"), - ], - tools_json, + &[user_message(question)], + tools, &ChatSessionOptions { assistant_turn: ChatAssistantTurn::ContinueContent(ChatAssistantPrefill { reasoning_content: PREFILLED_REASONING.to_owned(), diff --git a/llama-cpp-bindings-tests/src/get_weather_tools.rs b/llama-cpp-bindings-tests/src/get_weather_tools.rs new file mode 100644 index 00000000..4ab28b4c --- /dev/null +++ b/llama-cpp-bindings-tests/src/get_weather_tools.rs @@ -0,0 +1,17 @@ +use llama_cpp_bindings_types::chat_tool::ChatTool; +use serde_json::json; + +#[must_use] +pub fn get_weather_tools() -> Vec { + vec![ChatTool { + name: "get_weather".to_owned(), + description: "Get the current weather for a location".to_owned(), + parameters: Some(json!({ + "type": "object", + "properties": { + "location": {"type": "string", "description": "The city name"} + }, + "required": ["location"] + })), + }] +} diff --git a/llama-cpp-bindings-tests/src/get_weather_tools_json.rs b/llama-cpp-bindings-tests/src/get_weather_tools_json.rs deleted file mode 100644 index 71ce4753..00000000 --- a/llama-cpp-bindings-tests/src/get_weather_tools_json.rs +++ /dev/null @@ -1,16 +0,0 @@ -pub const GET_WEATHER_TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; diff --git a/llama-cpp-bindings-tests/src/ingested_generation.rs b/llama-cpp-bindings-tests/src/ingested_generation.rs index cb3f3193..1cfc904e 100644 --- a/llama-cpp-bindings-tests/src/ingested_generation.rs +++ b/llama-cpp-bindings-tests/src/ingested_generation.rs @@ -13,7 +13,7 @@ use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; /// Panics when tokenizing or classifying fails, or when the classifier ends the generation /// before the end-of-sequence token, or does not end it on that token. pub fn ingested_generation( - classifier: &mut SampledTokenClassifier<'_, '_>, + classifier: &mut SampledTokenClassifier<'_>, model: &LlamaModel, generated_text: &str, ) -> Vec { diff --git a/llama-cpp-bindings-tests/src/lib.rs b/llama-cpp-bindings-tests/src/lib.rs index f2a08626..369da0e1 100644 --- a/llama-cpp-bindings-tests/src/lib.rs +++ b/llama-cpp-bindings-tests/src/lib.rs @@ -8,7 +8,7 @@ pub mod closed_reasoning_chat_session; pub mod context_after_decoding_text; pub mod evaluated_multimodal_usage; pub mod fixtures_dir; -pub mod get_weather_tools_json; +pub mod get_weather_tools; pub mod ingested_generation; pub mod logits_left_by; pub mod media_question_chat_session; @@ -19,7 +19,9 @@ pub mod prime_memory_with; pub mod prompt_question; pub mod recognized_answer; pub mod single_token; +pub mod system_message; pub mod tokenize_prompt_with_synthetic_image; pub mod two_sequence_context; +pub mod user_message; pub mod user_turn_chat_session; pub mod weather_chat_session; diff --git a/llama-cpp-bindings-tests/src/media_question_chat_session.rs b/llama-cpp-bindings-tests/src/media_question_chat_session.rs index 1798f000..ba2963fe 100644 --- a/llama-cpp-bindings-tests/src/media_question_chat_session.rs +++ b/llama-cpp-bindings-tests/src/media_question_chat_session.rs @@ -1,6 +1,5 @@ -use anyhow::Result; - use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::error::chat_session_error::ChatSessionError; use llama_cpp_bindings::model::LlamaModel; use llama_cpp_bindings::mtmd::mtmd_default_marker::mtmd_default_marker; @@ -11,22 +10,22 @@ use crate::user_turn_chat_session::user_turn_chat_session; /// /// # Errors /// -/// Forwards message construction and chat session errors. +/// Forwards chat session errors. /// /// # Panics /// /// Panics when llama.cpp's media marker is not UTF-8. -pub fn media_question_chat_session<'model>( - model: &'model LlamaModel, +pub fn media_question_chat_session( + model: &LlamaModel, question: &str, -) -> Result> { +) -> Result { user_turn_chat_session( model, &format!( "{}{question}", mtmd_default_marker().expect("llama.cpp's media marker must be UTF-8") ), - "[]", + &[], true, ) } diff --git a/llama-cpp-bindings-tests/src/recognized_answer.rs b/llama-cpp-bindings-tests/src/recognized_answer.rs index 36fb5789..4e7a0906 100644 --- a/llama-cpp-bindings-tests/src/recognized_answer.rs +++ b/llama-cpp-bindings-tests/src/recognized_answer.rs @@ -10,12 +10,15 @@ use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; /// Panics when parsing fails or the parser does not recognize the answer. #[must_use] pub fn recognized_answer( - session: &ChatSession<'_>, + session: &ChatSession, answer: &str, completeness: ChatOutputCompleteness, ) -> ParsedChatMessage { session - .parse(&ChatOutput::from(answer), completeness) + .read(&ChatOutput::from(answer), completeness) + .expect("the session's parser must read the answer") + .parse() + .cloned() .expect("the session's parser must parse the answer") .into_recognized() .expect("the session's parser must recognize the answer") diff --git a/llama-cpp-bindings-tests/src/system_message.rs b/llama-cpp-bindings-tests/src/system_message.rs new file mode 100644 index 00000000..aa030e2f --- /dev/null +++ b/llama-cpp-bindings-tests/src/system_message.rs @@ -0,0 +1,11 @@ +use llama_cpp_bindings_types::chat_message::ChatMessage; +use llama_cpp_bindings_types::chat_message_content::ChatMessageContent; + +#[must_use] +pub fn system_message(content: &str) -> ChatMessage { + ChatMessage::Message { + role: "system".to_owned(), + content: ChatMessageContent::Text(content.to_owned()), + reasoning_content: None, + } +} diff --git a/llama-cpp-bindings-tests/src/user_message.rs b/llama-cpp-bindings-tests/src/user_message.rs new file mode 100644 index 00000000..b49ea53e --- /dev/null +++ b/llama-cpp-bindings-tests/src/user_message.rs @@ -0,0 +1,11 @@ +use llama_cpp_bindings_types::chat_message::ChatMessage; +use llama_cpp_bindings_types::chat_message_content::ChatMessageContent; + +#[must_use] +pub fn user_message(content: &str) -> ChatMessage { + ChatMessage::Message { + role: "user".to_owned(), + content: ChatMessageContent::Text(content.to_owned()), + reasoning_content: None, + } +} diff --git a/llama-cpp-bindings-tests/src/user_turn_chat_session.rs b/llama-cpp-bindings-tests/src/user_turn_chat_session.rs index 47fd7efb..5e1e8e0d 100644 --- a/llama-cpp-bindings-tests/src/user_turn_chat_session.rs +++ b/llama-cpp-bindings-tests/src/user_turn_chat_session.rs @@ -1,42 +1,40 @@ -use anyhow::Result; - use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings::chat_session_options::ChatSessionOptions; use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; +use llama_cpp_bindings::error::chat_session_error::ChatSessionError; use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; +use llama_cpp_bindings_types::chat_tool::ChatTool; + +use crate::user_message::user_message; -/// A chat session that answers one user turn, offering the tools of `tools_json`. +/// A chat session that answers one user turn, offering `tools`. /// /// # Errors /// -/// Forwards message construction and chat session errors. +/// Forwards chat session errors. /// /// # Panics /// /// Panics when the model has no chat templates. -pub fn user_turn_chat_session<'model>( - model: &'model LlamaModel, +pub fn user_turn_chat_session( + model: &LlamaModel, content: &str, - tools_json: &str, + tools: &[ChatTool], enable_thinking: bool, -) -> Result> { - Ok(ChatSession::new( +) -> Result { + ChatSession::new( model, model .chat_templates() .expect("the model must have chat templates"), - &[LlamaChatMessage::new( - "user".to_owned(), - content.to_owned(), - )?], - tools_json, + &[user_message(content)], + tools, &ChatSessionOptions { assistant_turn: ChatAssistantTurn::New, enable_thinking, parallel_tool_calls: false, tool_choice: ChatToolChoice::Auto, }, - )?) + ) } diff --git a/llama-cpp-bindings-tests/src/weather_chat_session.rs b/llama-cpp-bindings-tests/src/weather_chat_session.rs index 565fba00..48b58552 100644 --- a/llama-cpp-bindings-tests/src/weather_chat_session.rs +++ b/llama-cpp-bindings-tests/src/weather_chat_session.rs @@ -3,9 +3,9 @@ use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings::chat_session_options::ChatSessionOptions; use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; -use crate::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; +use crate::get_weather_tools::get_weather_tools; +use crate::user_message::user_message; /// A chat session that asks for the weather in Paris, offering the weather tool with /// `tool_choice`. @@ -14,18 +14,14 @@ use crate::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; /// /// Panics when the model's template cannot serve the request. #[must_use] -pub fn weather_chat_session(model: &LlamaModel, tool_choice: ChatToolChoice) -> ChatSession<'_> { +pub fn weather_chat_session(model: &LlamaModel, tool_choice: ChatToolChoice) -> ChatSession { ChatSession::new( model, model .chat_templates() .expect("the model must have chat templates"), - &[LlamaChatMessage::new( - "user".to_owned(), - "What is the weather in Paris?".to_owned(), - ) - .expect("the question holds no NUL byte")], - GET_WEATHER_TOOLS_JSON, + &[user_message("What is the weather in Paris?")], + &get_weather_tools(), &ChatSessionOptions { assistant_turn: ChatAssistantTurn::New, enable_thinking: false, diff --git a/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs b/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs index dd047390..ccc28c8d 100644 --- a/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs +++ b/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs @@ -1,7 +1,5 @@ use anyhow::Result; -use anyhow::bail; -use llama_cpp_bindings::chat_message_parse_outcome::ChatMessageParseOutcome; use llama_cpp_bindings::chat_output::ChatOutput; use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; @@ -18,17 +16,12 @@ use llama_cpp_test_harness_macros::llama_test; const GENERATED_REASONING: &str = "step one, step two"; const GENERATED_CONTENT: &str = "actual response"; -fn parsed_content_after_closed_reasoning(fixture: &LlamaFixture<'_>) -> Result { - let outcome = closed_reasoning_chat_session(fixture.model, "hello", "[]").parse( - &ChatOutput::from("hello world"), +fn parsed_content_after_closed_reasoning(fixture: &LlamaFixture<'_>) -> ParsedChatMessage { + recognized_answer( + &closed_reasoning_chat_session(fixture.model, "hello", &[]), + "hello world", ChatOutputCompleteness::Finished, - )?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!("expected Recognized for plain content; got Unrecognized"); - }; - - Ok(parsed) + ) } #[llama_test( @@ -49,17 +42,15 @@ fn parsed_content_after_closed_reasoning(fixture: &LlamaFixture<'_>) -> Result

, -) -> Result<()> { +) { assert_eq!( - parsed_content_after_closed_reasoning(fixture)?, + parsed_content_after_closed_reasoning(fixture), ParsedChatMessage::new( "\n\nhello world".to_owned(), "\nThe question is simple, so I answer it directly.\n".to_owned(), Vec::new() ) ); - - Ok(()) } #[llama_test( @@ -78,23 +69,19 @@ fn templates_that_keep_prefill_whitespace_parse_it_into_reasoning_and_content( n_batch = 128, n_ubatch = 64, )] -fn qwen_templates_parse_trimmed_prefill_into_reasoning_and_content( - fixture: &LlamaFixture<'_>, -) -> Result<()> { +fn qwen_templates_parse_trimmed_prefill_into_reasoning_and_content(fixture: &LlamaFixture<'_>) { assert_eq!( - parsed_content_after_closed_reasoning(fixture)?, + parsed_content_after_closed_reasoning(fixture), ParsedChatMessage::new( "hello world".to_owned(), "The question is simple, so I answer it directly.\n".to_owned(), Vec::new() ) ); - - Ok(()) } fn reasoning_parsed_before_the_end_tag(fixture: &LlamaFixture<'_>) -> Result { - let session = user_turn_chat_session(fixture.model, "hello", "[]", true)?; + let session = user_turn_chat_session(fixture.model, "hello", &[], true)?; Ok(recognized_answer( &session, @@ -198,13 +185,12 @@ fn parses_the_reasoning_before_the_end_tag_apart_from_the_content( n_ubatch = 64, )] fn parses_an_empty_answer_into_an_empty_message(fixture: &LlamaFixture<'_>) -> Result<()> { - let outcome = user_turn_chat_session(fixture.model, "hello", "[]", false)? - .parse(&ChatOutput::from(""), ChatOutputCompleteness::Finished)?; + let session = user_turn_chat_session(fixture.model, "hello", &[], false)?; - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!("expected Recognized for empty input; got Unrecognized"); - }; - assert_eq!(parsed, ParsedChatMessage::default()); + assert_eq!( + recognized_answer(&session, "", ChatOutputCompleteness::Finished), + ParsedChatMessage::default() + ); Ok(()) } @@ -218,16 +204,12 @@ fn parses_an_empty_answer_into_an_empty_message(fixture: &LlamaFixture<'_>) -> R n_ubatch = 64, )] fn parses_an_answer_with_a_nul_byte_as_content(fixture: &LlamaFixture<'_>) -> Result<()> { - let outcome = user_turn_chat_session(fixture.model, "hello", "[]", false)?.parse( - &ChatOutput::from("hello\0world"), - ChatOutputCompleteness::Finished, - )?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!("an answer with a NUL byte must be recognized as content"); - }; + let session = user_turn_chat_session(fixture.model, "hello", &[], false)?; - assert_eq!(parsed.content, "hello\0world"); + assert_eq!( + recognized_answer(&session, "hello\0world", ChatOutputCompleteness::Finished).content, + "hello\0world" + ); Ok(()) } @@ -241,16 +223,13 @@ fn parses_an_answer_with_a_nul_byte_as_content(fixture: &LlamaFixture<'_>) -> Re n_ubatch = 64, )] fn qwen35_chat_session_parses_every_answer_it_is_given(fixture: &LlamaFixture<'_>) -> Result<()> { - let session = user_turn_chat_session(fixture.model, "hello", "[]", false)?; + let session = user_turn_chat_session(fixture.model, "hello", &[], false)?; for answer in ["the first answer", "the second answer"] { - let ChatMessageParseOutcome::Recognized(parsed) = - session.parse(&ChatOutput::from(answer), ChatOutputCompleteness::Finished)? - else { - bail!("expected {answer:?} to be recognized as content"); - }; - - assert_eq!(parsed.content, answer); + assert_eq!( + recognized_answer(&session, answer, ChatOutputCompleteness::Finished).content, + answer + ); } Ok(()) @@ -282,12 +261,35 @@ fn every_parsed_field_refuses_text_that_is_not_utf8(fixture: &LlamaFixture<'_>) assert_eq!( fault_points.map(|fault_point| { + let reading = session + .read(&answer, ChatOutputCompleteness::Finished) + .expect("the weather answer must be read"); let _fault = ArmedFault::at(fault_point); - session - .parse(&answer, ChatOutputCompleteness::Finished) - .err() + reading.parse().err().cloned() }), fault_points.map(|_fault_point| Some(ParseChatMessageError::ParsedTextNotUtf8(not_utf8))) ); } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn a_reading_maps_its_parse_once_and_keeps_it(fixture: &LlamaFixture<'_>) -> Result<()> { + let reading = user_turn_chat_session(fixture.model, "hello", &[], false)?.read( + &ChatOutput::from("Hello there."), + ChatOutputCompleteness::Finished, + )?; + let first_parse = reading.parse().cloned(); + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_READING_PARSE_THROWS); + + assert_eq!(reading.parse().cloned(), first_parse); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs b/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs index c9b2f8e8..89e69e04 100644 --- a/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs +++ b/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs @@ -4,8 +4,11 @@ use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings::chat_session_options::ChatSessionOptions; use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; -use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; +use llama_cpp_bindings_tests::user_message::user_message; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_bindings_types::chat_message::ChatMessage; +use llama_cpp_bindings_types::chat_message_content::ChatMessageContent; +use llama_cpp_bindings_types::chat_message_tool_call::ChatMessageToolCall; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -61,7 +64,7 @@ use llama_cpp_test_harness_macros::llama_test; n_ubatch = 64, )] fn chat_session_renders_the_user_message_into_the_prompt(fixture: &LlamaFixture<'_>) -> Result<()> { - let prompt = user_turn_chat_session(fixture.model, "hello", "[]", true)?.prompt()?; + let prompt = user_turn_chat_session(fixture.model, "hello", &[], true)?.prompt()?; assert!( prompt.contains("hello"), @@ -105,7 +108,7 @@ fn chat_session_renders_the_user_message_into_the_prompt(fixture: &LlamaFixture< )] fn chat_session_renders_long_messages_in_full(fixture: &LlamaFixture<'_>) -> Result<()> { let long_content = "a".repeat(2000); - let prompt = user_turn_chat_session(fixture.model, &long_content, "[]", true)?.prompt()?; + let prompt = user_turn_chat_session(fixture.model, &long_content, &[], true)?.prompt()?; assert!( prompt.contains(&long_content), @@ -125,7 +128,7 @@ fn chat_session_renders_long_messages_in_full(fixture: &LlamaFixture<'_>) -> Res fn qwen35_generation_prompt_opens_the_assistant_turn_with_thinking( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let session = user_turn_chat_session(fixture.model, "hello", "[]", true)?; + let session = user_turn_chat_session(fixture.model, "hello", &[], true)?; assert_eq!( session.generation_prompt()?, @@ -149,11 +152,8 @@ fn qwen35_omitted_assistant_turn_renders_only_the_messages( let session = ChatSession::new( fixture.model, fixture.model.chat_templates()?, - &[LlamaChatMessage::new( - "user".to_owned(), - "hello".to_owned(), - )?], - "[]", + &[user_message("hello")], + &[], &ChatSessionOptions { assistant_turn: ChatAssistantTurn::Omitted, enable_thinking: true, @@ -166,3 +166,56 @@ fn qwen35_omitted_assistant_turn_renders_only_the_messages( Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_renders_a_tool_call_history_with_its_reasoning_and_tool_response( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = ChatSession::new( + fixture.model, + fixture.model.chat_templates()?, + &[ + user_message("What is the weather in Paris?"), + ChatMessage::AssistantToolCalls { + content: ChatMessageContent::Text(String::new()), + reasoning_content: Some("I need the weather tool.".to_owned()), + tool_calls: vec![ChatMessageToolCall { + id: "call_weather".to_owned(), + name: "get_weather".to_owned(), + arguments: r#"{"location":"Paris"}"#.to_owned(), + }], + }, + ChatMessage::ToolResult { + content: ChatMessageContent::Text("sunny".to_owned()), + tool_call_id: "call_weather".to_owned(), + }, + ], + &[], + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::Omitted, + enable_thinking: true, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + )?; + + assert_eq!( + session.prompt()?, + concat!( + "<|im_start|>user\nWhat is the weather in Paris?<|im_end|>\n", + "<|im_start|>assistant\n\nI need the weather tool.\n\n\n", + "\n\n\nParis\n\n", + "\n<|im_end|>\n", + "<|im_start|>user\n\nsunny\n<|im_end|>\n", + ) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs index 4098cad0..7c0f6be0 100644 --- a/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs +++ b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs @@ -8,11 +8,10 @@ use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings::chat_session_options::ChatSessionOptions; use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; use llama_cpp_bindings::error::chat_session_error::ChatSessionError; -use llama_cpp_bindings::error::new_llama_chat_message_error::NewLlamaChatMessageError; use llama_cpp_bindings::error::string_to_token_error::StringToTokenError; -use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; +use llama_cpp_bindings::mtmd::mtmd_default_marker::mtmd_default_marker; use llama_cpp_bindings_tests::media_question_chat_session::media_question_chat_session; -use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_bindings_tests::user_message::user_message; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -24,100 +23,14 @@ use llama_cpp_test_harness_macros::llama_test; n_batch = 64, n_ubatch = 64, )] -fn chat_message_reports_the_nul_byte_position_after_the_media_marker( +fn a_question_holding_a_nul_character_is_rendered_into_the_prompt_after_the_media_marker( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let Err(failure) = media_question_chat_session(fixture.model, "describe\0this") else { - bail!("an interior NUL byte in the question must refuse the message"); - }; - let NewLlamaChatMessageError::NulError(nul_error) = - failure.downcast::()?; - - assert_eq!(nul_error.nul_position(), "<__media__>describe".len()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = -1, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn chat_session_refuses_a_tool_missing_its_function_name(fixture: &LlamaFixture<'_>) -> Result<()> { - let tools_json = r#"[{"type":"function","function":{"description":"reports the weather"}}]"#; - let result = user_turn_chat_session(fixture.model, "hello", tools_json, false); - let Err(failure) = result else { - bail!("a tool without a function name must refuse the chat session"); - }; - - assert_eq!( - failure.downcast::()?, - ChatSessionError::ToolsInvalid { - message: "Failed to parse tools: [json.exception.out_of_range.403] key 'name' not \ - found; tools = [\n {\n \"type\": \"function\",\n \"function\": {\n \ - \"description\": \"reports the weather\"\n }\n }\n]" - .to_owned(), - } - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = -1, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn chat_session_refuses_tools_with_a_nul_byte(fixture: &LlamaFixture<'_>) { - assert_eq!( - user_turn_chat_session(fixture.model, "hello", "[\0]", false) - .err() - .and_then(|failure| failure.downcast::().ok()), - CString::new("[\0]") - .err() - .map(ChatSessionError::RequestContainsNul) - ); -} + let prompt = media_question_chat_session(fixture.model, "describe\0this")?.prompt()?; -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = -1, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn chat_session_refuses_a_message_that_is_not_utf8(fixture: &LlamaFixture<'_>) -> Result<()> { - let not_utf8 = CString::new([0xff])?; - let message = LlamaChatMessage { - role: CString::new("user")?, - content: not_utf8.clone(), - }; - - assert_eq!( - ChatSession::new( - fixture.model, - fixture.model.chat_templates()?, - &[message], - "[]", - &ChatSessionOptions { - assistant_turn: ChatAssistantTurn::New, - enable_thinking: false, - parallel_tool_calls: false, - tool_choice: ChatToolChoice::Auto, - }, - ) - .err(), - not_utf8 - .to_str() - .err() - .map(ChatSessionError::MessageNotUtf8) + assert!( + prompt.contains(&format!("{}describe\0this", mtmd_default_marker()?)), + "{prompt:?}" ); Ok(()) @@ -137,11 +50,8 @@ fn tokenizing_a_prompt_whose_reasoning_prefill_holds_a_nul_character_is_refused( let session = ChatSession::new( fixture.model, fixture.model.chat_templates()?, - &[LlamaChatMessage::new( - "user".to_owned(), - "hello".to_owned(), - )?], - "[]", + &[user_message("hello")], + &[], &ChatSessionOptions { assistant_turn: ChatAssistantTurn::ContinueReasoning("a\0b".to_owned()), enable_thinking: true, @@ -158,7 +68,7 @@ fn tokenizing_a_prompt_whose_reasoning_prefill_holds_a_nul_character_is_refused( }; assert_eq!( - session.tokenize_prompt().err(), + session.tokenize_prompt(fixture.model).err(), Some(ChatSessionError::PromptTokenization( StringToTokenError::NulError(nul_error) )) @@ -181,11 +91,8 @@ fn a_conversation_the_template_raises_on_is_refused_with_the_template_message( let result = ChatSession::new( fixture.model, fixture.model.chat_templates()?, - &[ - LlamaChatMessage::new("user".to_owned(), "hello".to_owned())?, - LlamaChatMessage::new("user".to_owned(), "are you there?".to_owned())?, - ], - "[]", + &[user_message("hello"), user_message("are you there?")], + &[], &ChatSessionOptions { assistant_turn: ChatAssistantTurn::New, enable_thinking: false, diff --git a/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs b/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs index 95125263..c7cd74fd 100644 --- a/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs +++ b/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs @@ -15,6 +15,8 @@ use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; +const MALFORMED_QWEN_XML_TOOL_CALL: &str = "garbage"; + fn not_utf8() -> Utf8Error { String::from_utf8(vec![0xff]) .expect_err("a lone 0xff byte is not UTF-8") @@ -54,7 +56,7 @@ fn every_prompt_reader_refuses_prompt_text_that_is_not_utf8(fixture: &LlamaFixtu [ session.prompt_parts().err(), session.prompt().err(), - session.tokenize_prompt().err(), + session.tokenize_prompt(fixture.model).err(), ], [(); 3].map(|()| Some(ChatSessionError::SessionTextNotUtf8(not_utf8()))) ); @@ -77,7 +79,7 @@ fn grammar_triggers_refuse_a_trigger_value_that_is_not_utf8(fixture: &LlamaFixtu assert_eq!( [ session.grammar_triggers().err(), - session.grammar_sampler().err(), + session.grammar_sampler(fixture.model).err(), ], [(); 2].map(|()| Some(ChatSessionError::SessionTextNotUtf8(not_utf8()))) ); @@ -124,7 +126,7 @@ fn the_grammar_sampler_refuses_a_grammar_that_is_not_utf8(fixture: &LlamaFixture ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_NOT_UTF8); assert_eq!( - session.grammar_sampler().err(), + session.grammar_sampler(fixture.model).err(), Some(ChatSessionError::SessionTextNotUtf8(not_utf8())) ); } @@ -144,7 +146,7 @@ fn the_grammar_sampler_reports_a_grammar_llama_cpp_cannot_load(fixture: &LlamaFi assert!(!session.grammar_is_lazy()); assert_eq!( - session.grammar_sampler().err(), + session.grammar_sampler(fixture.model).err(), Some(ChatSessionError::Grammar(GrammarError::Reported { message: injected_fault_message(), })) @@ -167,7 +169,7 @@ fn the_grammar_sampler_reports_a_generation_prompt_token_the_grammar_cannot_acce assert_ne!(session.generation_prompt_tokens(), []); assert_eq!( - session.grammar_sampler().err(), + session.grammar_sampler(fixture.model).err(), Some(ChatSessionError::GrammarPrefill( SamplerAcceptError::GrammarStateCorrupted { message: injected_fault_message(), @@ -190,7 +192,7 @@ fn the_grammar_sampler_reports_a_trigger_word_llama_cpp_cannot_escape(fixture: & assert!(session.grammar_is_lazy()); assert_eq!( - session.grammar_sampler().err(), + session.grammar_sampler(fixture.model).err(), Some(ChatSessionError::Grammar(GrammarError::Reported { message: injected_fault_message(), })) @@ -212,7 +214,7 @@ fn the_grammar_sampler_reports_a_lazy_grammar_llama_cpp_cannot_load(fixture: &Ll ); assert_eq!( - session.grammar_sampler().err(), + session.grammar_sampler(fixture.model).err(), Some(ChatSessionError::Grammar(GrammarError::Reported { message: injected_fault_message(), })) @@ -227,14 +229,14 @@ fn the_grammar_sampler_reports_a_lazy_grammar_llama_cpp_cannot_load(fixture: &Ll n_batch = 128, n_ubatch = 64, )] -fn parsing_an_answer_reports_a_parser_llama_cpp_fails(fixture: &LlamaFixture<'_>) { +fn reading_an_answer_reports_a_parse_llama_cpp_fails(fixture: &LlamaFixture<'_>) { let session = weather_chat_session(fixture.model, ChatToolChoice::Auto); let _fault = - ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_PARSE_THROWS); + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_READ_THROWS); assert_eq!( session - .parse( + .read( &ChatOutput::from("Checking."), ChatOutputCompleteness::Finished, ) @@ -244,3 +246,56 @@ fn parsing_an_answer_reports_a_parser_llama_cpp_fails(fixture: &LlamaFixture<'_> }) ); } + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn parsing_a_reading_reports_a_mapping_llama_cpp_fails(fixture: &LlamaFixture<'_>) { + let reading = weather_chat_session(fixture.model, ChatToolChoice::Auto) + .read( + &ChatOutput::from("Checking."), + ChatOutputCompleteness::Finished, + ) + .expect("the answer must be read"); + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_READING_PARSE_THROWS); + + assert_eq!( + reading.parse().err(), + Some(&ParseChatMessageError::Reported { + message: injected_fault_message(), + }) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn reading_an_answer_that_does_not_match_the_format_refuses_a_format_name_that_is_not_utf8( + fixture: &LlamaFixture<'_>, +) { + let session = weather_chat_session(fixture.model, ChatToolChoice::Auto); + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_READING_FORMAT_NAME_NOT_UTF8, + ); + + assert_eq!( + session + .read( + &ChatOutput::from(MALFORMED_QWEN_XML_TOOL_CALL), + ChatOutputCompleteness::Finished, + ) + .err(), + Some(ParseChatMessageError::FormatNameNotUtf8(not_utf8())) + ); +} diff --git a/llama-cpp-bindings-tests/tests/chat_template_override.rs b/llama-cpp-bindings-tests/tests/chat_template_override.rs index 9cb05a47..22665545 100644 --- a/llama-cpp-bindings-tests/tests/chat_template_override.rs +++ b/llama-cpp-bindings-tests/tests/chat_template_override.rs @@ -12,14 +12,14 @@ use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; use llama_cpp_bindings::error::chat_templates_error::ChatTemplatesError; use llama_cpp_bindings::model::LlamaModel; use llama_cpp_bindings::model::add_bos::AddBos; -use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; use llama_cpp_bindings::model::llama_chat_template::LlamaChatTemplate; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; -use llama_cpp_bindings_tests::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; +use llama_cpp_bindings_tests::get_weather_tools::get_weather_tools; use llama_cpp_bindings_tests::ingested_generation::ingested_generation; use llama_cpp_bindings_tests::paris_weather_tool_call::paris_weather_tool_call; use llama_cpp_bindings_tests::recognized_answer::recognized_answer; +use llama_cpp_bindings_tests::user_message::user_message; use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -35,18 +35,12 @@ fn llama_cpp_deepseek_r1_template() -> Result { Ok(LlamaChatTemplate::new(&fs::read_to_string(path)?)?) } -fn weather_session<'model>( - model: &'model LlamaModel, - templates: &ChatTemplates, -) -> Result> { +fn weather_session(model: &LlamaModel, templates: &ChatTemplates) -> Result { Ok(ChatSession::new( model, templates, - &[LlamaChatMessage::new( - "user".to_owned(), - "What is the weather in Paris?".to_owned(), - )?], - GET_WEATHER_TOOLS_JSON, + &[user_message("What is the weather in Paris?")], + &get_weather_tools(), &ChatSessionOptions { assistant_turn: ChatAssistantTurn::New, enable_thinking: true, @@ -109,7 +103,7 @@ fn deepseek_r1_served_with_the_llama_cpp_template_streams_its_tool_call_tokens_a let templates = ChatTemplates::with_override(fixture.model, &llama_cpp_deepseek_r1_template()?)?; let session = weather_session(fixture.model, &templates)?; - let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let mut classifier = SampledTokenClassifier::for_chat_session(fixture.model, session)?; ingested_generation( &mut classifier, @@ -149,3 +143,28 @@ fn an_empty_template_override_is_refused(fixture: &LlamaFixture<'_>) -> Result<( Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 1024, + n_batch = 128, + n_ubatch = 64, +)] +fn a_template_override_llama_cpp_cannot_parse_reports_its_parse_error( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + assert_eq!( + ChatTemplates::with_override( + fixture.model, + &LlamaChatTemplate::new("{{invalid jinja template")? + ) + .err(), + Some(ChatTemplatesError::Reported { + message: "parser: Parser Error: Expected }} (Got jinja)\n...{{invalid jinja template...\n ^".to_owned(), + }) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs index 5e0688f8..b6675e5b 100644 --- a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs +++ b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs @@ -9,11 +9,11 @@ use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::add_bos::AddBos; -use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; +use llama_cpp_bindings_tests::user_message::user_message; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -177,15 +177,12 @@ fn chat_completion_usage_matches_the_classified_tokens(fixture: &LlamaFixture<'_ (*fixture.context_params).into_llama_context_params(), )?; - let messages = [LlamaChatMessage::new( - "user".to_string(), - "Hello! How are you?".to_string(), - )?]; + let messages = [user_message("Hello! How are you?")]; let session = ChatSession::new( model, model.chat_templates()?, &messages, - "[]", + &[], &ChatSessionOptions { assistant_turn: ChatAssistantTurn::New, enable_thinking: true, @@ -194,8 +191,8 @@ fn chat_completion_usage_matches_the_classified_tokens(fixture: &LlamaFixture<'_ }, )?; - let mut classifier = SampledTokenClassifier::for_chat_session(&session); - let tokens = session.tokenize_prompt()?; + let tokens = session.tokenize_prompt(fixture.model)?; + let mut classifier = SampledTokenClassifier::for_chat_session(fixture.model, session)?; let prompt_token_count = u64::try_from(tokens.len())?; let mut batch = LlamaBatch::new(ContextLimits::of(&context)); diff --git a/llama-cpp-bindings-tests/tests/generation_end_classification.rs b/llama-cpp-bindings-tests/tests/generation_end_classification.rs index e6db6af1..b24290c5 100644 --- a/llama-cpp-bindings-tests/tests/generation_end_classification.rs +++ b/llama-cpp-bindings-tests/tests/generation_end_classification.rs @@ -32,8 +32,8 @@ fn qwen35_classifier_ends_generation_without_emitting_the_end_of_generation_toke ) -> Result<()> { const GENERATED_TEXT: &str = "The answer is four."; - let session = user_turn_chat_session(fixture.model, "What is 2 + 2?", "[]", false)?; - let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let session = user_turn_chat_session(fixture.model, "What is 2 + 2?", &[], false)?; + let mut classifier = SampledTokenClassifier::for_chat_session(fixture.model, session)?; assert_eq!( visible_text(&ingested_generation( @@ -69,8 +69,8 @@ fn an_answer_stopped_inside_unclosed_reasoning_releases_its_unsettled_tail_as_co const SETTLED_REASONING: &str = "Let me think"; const UNSETTLED_TAIL: &str = " Result<()> { const GENERATED_TEXT: &str = r#"{"name": "get_weather", "arguments": {"location": "Paris"}}"#; - let session = user_turn_chat_session(fixture.model, "What is 2 + 2?", "[]", false)?; - let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let session = user_turn_chat_session(fixture.model, "What is 2 + 2?", &[], false)?; + let mut classifier = SampledTokenClassifier::for_chat_session(fixture.model, session)?; let outcomes = ingested_generation(&mut classifier, fixture.model, GENERATED_TEXT); assert_eq!( diff --git a/llama-cpp-bindings-tests/tests/grammar_sampler.rs b/llama-cpp-bindings-tests/tests/grammar_sampler.rs index 9dc7b719..a1f1d27b 100644 --- a/llama-cpp-bindings-tests/tests/grammar_sampler.rs +++ b/llama-cpp-bindings-tests/tests/grammar_sampler.rs @@ -145,11 +145,11 @@ fn grammar_constrained_chat_answer_is_yes_content_that_ends_generation( (*fixture.context_params).into_llama_context_params(), )?; - let session = closed_reasoning_chat_session(model, "Is the sky blue? yes or no", "[]"); - let tokens = session.tokenize_prompt()?; + let session = closed_reasoning_chat_session(model, "Is the sky blue? yes or no", &[]); + let tokens = session.tokenize_prompt(fixture.model)?; let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let mut classifier = SampledTokenClassifier::for_chat_session(fixture.model, session)?; classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; context.decode(&batch)?; diff --git a/llama-cpp-bindings-tests/tests/image_question_answering.rs b/llama-cpp-bindings-tests/tests/image_question_answering.rs index 7cba5800..2b8a4ab9 100644 --- a/llama-cpp-bindings-tests/tests/image_question_answering.rs +++ b/llama-cpp-bindings-tests/tests/image_question_answering.rs @@ -78,7 +78,7 @@ fn a_question_about_an_image_is_answered_with_its_image_tokens_counted_as_input( "vision input must produce at least one image chunk" ); - let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let mut classifier = SampledTokenClassifier::for_chat_session(fixture.model, session)?; let n_past = classifier .eval_multimodal_chunks( &chunks, diff --git a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs index 0895dced..6f25e909 100644 --- a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs +++ b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs @@ -27,8 +27,8 @@ fn json_schema_constrained_answer(fixture: &LlamaFixture<'_>) -> Result { )?; let session = - closed_reasoning_chat_session(model, "What is 2+2? Respond with a JSON object.", "[]"); - let tokens = session.tokenize_prompt()?; + closed_reasoning_chat_session(model, "What is 2+2? Respond with a JSON object.", &[]); + let tokens = session.tokenize_prompt(fixture.model)?; let mut batch = LlamaBatch::new(ContextLimits::of(&context)); let mut classifier = SampledTokenClassifier::undeterminable(model); diff --git a/llama-cpp-bindings-tests/tests/main.rs b/llama-cpp-bindings-tests/tests/main.rs index 8eb28ab1..21ca45aa 100644 --- a/llama-cpp-bindings-tests/tests/main.rs +++ b/llama-cpp-bindings-tests/tests/main.rs @@ -70,6 +70,7 @@ mod special_tokens; mod state_snapshots; mod token_attributes; mod token_candidates; +mod token_evaluation_accounting; mod tokenization; mod tool_call_classification; mod tool_call_grammar; diff --git a/llama-cpp-bindings-tests/tests/multimodal_audio.rs b/llama-cpp-bindings-tests/tests/multimodal_audio.rs index 2db5f411..c38ea71f 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_audio.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_audio.rs @@ -9,7 +9,6 @@ use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::eval_multimodal_chunks_params::EvalMultimodalChunksParams; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; use llama_cpp_bindings::mtmd::mtmd_default_marker::mtmd_default_marker; use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; @@ -19,6 +18,8 @@ use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; use llama_cpp_bindings_tests::evaluated_multimodal_usage::EvaluatedMultimodalUsage; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; +use llama_cpp_bindings_tests::system_message::system_message; +use llama_cpp_bindings_tests::user_message::user_message; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -29,7 +30,7 @@ fn evaluate_audio_prompt<'model, TOutcome>( fixture: &LlamaFixture<'model>, audio_file_name: &str, after_evaluation: impl FnOnce( - &mut SampledTokenClassifier<'_, 'model>, + &mut SampledTokenClassifier<'model>, &mut LlamaContext<'model>, ChunkTokenBreakdown, i32, @@ -49,14 +50,14 @@ fn evaluate_audio_prompt<'model, TOutcome>( let marker = mtmd_default_marker()?; let messages = [ - LlamaChatMessage::new("system".to_string(), TRANSCRIBE_SYSTEM_PROMPT.to_string())?, - LlamaChatMessage::new("user".to_string(), marker.to_owned())?, + system_message(TRANSCRIBE_SYSTEM_PROMPT), + user_message(marker), ]; let session = ChatSession::new( model, model.chat_templates()?, &messages, - "[]", + &[], &ChatSessionOptions { assistant_turn: ChatAssistantTurn::New, enable_thinking: true, @@ -88,7 +89,7 @@ fn evaluate_audio_prompt<'model, TOutcome>( "tokenization should produce at least one chunk" ); - let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let mut classifier = SampledTokenClassifier::for_chat_session(fixture.model, session)?; let n_past = classifier .eval_multimodal_chunks( &chunks, diff --git a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs index d402d824..41538972 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs @@ -10,7 +10,6 @@ use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::eval_multimodal_chunks_params::EvalMultimodalChunksParams; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; use llama_cpp_bindings::mtmd::mtmd_context::MtmdContext; use llama_cpp_bindings::mtmd::mtmd_default_marker::mtmd_default_marker; @@ -20,6 +19,7 @@ use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; +use llama_cpp_bindings_tests::user_message::user_message; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -27,16 +27,16 @@ const MAX_GENERATED_TOKENS: i32 = 512; const DESCRIBE_INSTRUCTION: &str = "Describe the animal shown in the image, then write the exact words spoken in the audio."; -fn describe_image_and_audio_chat_session(model: &LlamaModel) -> Result> { +fn describe_image_and_audio_chat_session(model: &LlamaModel) -> Result { let marker = mtmd_default_marker()?; let user_content = format!("Image: {marker}\nAudio: {marker}\n{DESCRIBE_INSTRUCTION}"); - let messages = [LlamaChatMessage::new("user".to_string(), user_content)?]; + let messages = [user_message(&user_content)]; Ok(ChatSession::new( model, model.chat_templates()?, &messages, - "[]", + &[], &ChatSessionOptions { assistant_turn: ChatAssistantTurn::New, enable_thinking: false, @@ -118,7 +118,7 @@ fn an_image_and_audio_prompt_is_answered_with_both_media_counted_as_input( .with_context(|| "unable to create llama context")?; let n_batch = i32::try_from(context.n_batch())?; - let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let mut classifier = SampledTokenClassifier::for_chat_session(fixture.model, session)?; let n_past = classifier .eval_multimodal_chunks( &chunks, diff --git a/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs index 3e0fc4e9..2a8b4add 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs @@ -10,7 +10,6 @@ use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::eval_multimodal_chunks_params::EvalMultimodalChunksParams; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; use llama_cpp_bindings::mtmd::mtmd_default_marker::mtmd_default_marker; use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; @@ -20,6 +19,8 @@ use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; use llama_cpp_bindings_tests::media_question_chat_session::media_question_chat_session; use llama_cpp_bindings_tests::prompt_question::PROMPT_QUESTION; +use llama_cpp_bindings_tests::system_message::system_message; +use llama_cpp_bindings_tests::user_message::user_message; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -64,7 +65,7 @@ fn gemma4_classifier_emits_reasoning_for_multimodal_thinking_prompt( let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let mut classifier = SampledTokenClassifier::for_chat_session(fixture.model, session)?; let n_past = classifier.eval_multimodal_chunks( &chunks, mtmd_ctx, @@ -148,8 +149,7 @@ fn mistral3_classifier_emits_reasoning_for_multimodal_thinking_prompt( let marker = mtmd_default_marker()?; let messages = [ - LlamaChatMessage::new( - "system".to_owned(), + system_message( "# HOW YOU SHOULD THINK AND ANSWER\n\n\ First draft your thinking process (inner monologue) until you arrive at a response. \ Format your response using Markdown, and use LaTeX for any mathematical equations. \ @@ -157,16 +157,15 @@ fn mistral3_classifier_emits_reasoning_for_multimodal_thinking_prompt( Your thinking process must follow the template below:\ [THINK]Your thoughts or/and draft, like working through an exercise on scratch paper. \ Be as casual and as long as you want until you are confident to generate the response \ - to the user.[/THINK]Here, provide a self-contained response." - .to_owned(), - )?, - LlamaChatMessage::new("user".to_owned(), format!("{marker}{PROMPT_QUESTION}"))?, + to the user.[/THINK]Here, provide a self-contained response.", + ), + user_message(&format!("{marker}{PROMPT_QUESTION}")), ]; let session = ChatSession::new( model, model.chat_templates()?, &messages, - "[]", + &[], &ChatSessionOptions { assistant_turn: ChatAssistantTurn::New, enable_thinking: true, @@ -182,7 +181,7 @@ fn mistral3_classifier_emits_reasoning_for_multimodal_thinking_prompt( let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let mut classifier = SampledTokenClassifier::for_chat_session(fixture.model, session)?; let n_past = classifier.eval_multimodal_chunks( &chunks, mtmd_ctx, @@ -283,7 +282,7 @@ fn qwen_classifier_emits_reasoning_for_multimodal_thinking_prompt( let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let mut classifier = SampledTokenClassifier::for_chat_session(fixture.model, session)?; let n_past = classifier.eval_multimodal_chunks( &chunks, mtmd_ctx, diff --git a/llama-cpp-bindings-tests/tests/reasoning_classification.rs b/llama-cpp-bindings-tests/tests/reasoning_classification.rs index 0c472378..35aa26d1 100644 --- a/llama-cpp-bindings-tests/tests/reasoning_classification.rs +++ b/llama-cpp-bindings-tests/tests/reasoning_classification.rs @@ -4,10 +4,10 @@ use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings::chat_session_options::ChatSessionOptions; use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; -use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::chat_answer_generation::ChatAnswerGeneration; use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; +use llama_cpp_bindings_tests::user_message::user_message; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -28,16 +28,16 @@ const MAX_REASONING_ANSWER_TOKENS: i32 = 512; fn deepseek_r1_8b_answers_without_reasoning_after_its_reasoning_is_closed( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let session = closed_reasoning_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]"); + let session = closed_reasoning_chat_session(fixture.model, ARITHMETIC_QUESTION, &[]); let answer = ChatAnswerGeneration { fixture, - session: &session, + session, sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_ANSWER_TOKENS, } .run(); - answer.assert_content_without_reasoning(&session); + answer.assert_content_without_reasoning(); Ok(()) } @@ -53,16 +53,16 @@ fn deepseek_r1_8b_answers_without_reasoning_after_its_reasoning_is_closed( fn deepseek_r1_8b_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]", true)?; + let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, &[], true)?; let answer = ChatAnswerGeneration { fixture, - session: &session, + session, sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, } .run(); - answer.assert_reasoning_matches_parse(&session, "\n"); + answer.assert_reasoning_matches_parse("\n"); Ok(()) } @@ -78,16 +78,16 @@ fn deepseek_r1_8b_reasoning_streams_match_the_parsed_answer_when_thinking_is_ena fn gemma4_answers_without_reasoning_when_thinking_is_disabled( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let session = user_turn_chat_session(fixture.model, SINGLE_WORD_INSTRUCTION, "[]", false)?; + let session = user_turn_chat_session(fixture.model, SINGLE_WORD_INSTRUCTION, &[], false)?; let answer = ChatAnswerGeneration { fixture, - session: &session, + session, sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_ANSWER_TOKENS, } .run(); - answer.assert_content_without_reasoning(&session); + answer.assert_content_without_reasoning(); Ok(()) } @@ -106,11 +106,8 @@ fn gemma4_reasoning_streams_match_the_parsed_answer_when_reasoning_is_prefilled( let session = ChatSession::new( fixture.model, fixture.model.chat_templates()?, - &[LlamaChatMessage::new( - "user".to_owned(), - SINGLE_WORD_INSTRUCTION.to_owned(), - )?], - "[]", + &[user_message(SINGLE_WORD_INSTRUCTION)], + &[], &ChatSessionOptions { assistant_turn: ChatAssistantTurn::ContinueReasoning( "The user wants one word.".to_owned(), @@ -122,13 +119,13 @@ fn gemma4_reasoning_streams_match_the_parsed_answer_when_reasoning_is_prefilled( )?; let answer = ChatAnswerGeneration { fixture, - session: &session, + session, sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, } .run(); - answer.assert_reasoning_matches_parse(&session, "The user wants one word."); + answer.assert_reasoning_matches_parse("The user wants one word."); Ok(()) } @@ -144,16 +141,16 @@ fn gemma4_reasoning_streams_match_the_parsed_answer_when_reasoning_is_prefilled( fn glm47_answers_without_reasoning_when_thinking_is_disabled( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]", false)?; + let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, &[], false)?; let answer = ChatAnswerGeneration { fixture, - session: &session, + session, sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_ANSWER_TOKENS, } .run(); - answer.assert_content_without_reasoning(&session); + answer.assert_content_without_reasoning(); Ok(()) } @@ -169,16 +166,16 @@ fn glm47_answers_without_reasoning_when_thinking_is_disabled( fn glm47_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]", true)?; + let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, &[], true)?; let answer = ChatAnswerGeneration { fixture, - session: &session, + session, sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, } .run(); - answer.assert_reasoning_matches_parse(&session, ""); + answer.assert_reasoning_matches_parse(""); Ok(()) } @@ -194,16 +191,16 @@ fn glm47_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( fn mistral3_answers_without_reasoning_after_its_reasoning_is_closed( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let session = closed_reasoning_chat_session(fixture.model, SINGLE_WORD_INSTRUCTION, "[]"); + let session = closed_reasoning_chat_session(fixture.model, SINGLE_WORD_INSTRUCTION, &[]); let answer = ChatAnswerGeneration { fixture, - session: &session, + session, sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_ANSWER_TOKENS, } .run(); - answer.assert_content_without_reasoning(&session); + answer.assert_content_without_reasoning(); Ok(()) } @@ -219,16 +216,16 @@ fn mistral3_answers_without_reasoning_after_its_reasoning_is_closed( fn mistral3_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let session = user_turn_chat_session(fixture.model, SINGLE_WORD_INSTRUCTION, "[]", true)?; + let session = user_turn_chat_session(fixture.model, SINGLE_WORD_INSTRUCTION, &[], true)?; let answer = ChatAnswerGeneration { fixture, - session: &session, + session, sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, } .run(); - answer.assert_reasoning_matches_parse(&session, ""); + answer.assert_reasoning_matches_parse(""); Ok(()) } @@ -244,16 +241,16 @@ fn mistral3_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( fn qwen35_answers_without_reasoning_when_thinking_is_disabled( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]", false)?; + let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, &[], false)?; let answer = ChatAnswerGeneration { fixture, - session: &session, + session, sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_ANSWER_TOKENS, } .run(); - answer.assert_content_without_reasoning(&session); + answer.assert_content_without_reasoning(); Ok(()) } @@ -269,16 +266,16 @@ fn qwen35_answers_without_reasoning_when_thinking_is_disabled( fn qwen35_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]", true)?; + let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, &[], true)?; let answer = ChatAnswerGeneration { fixture, - session: &session, + session, sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, } .run(); - answer.assert_reasoning_matches_parse(&session, ""); + answer.assert_reasoning_matches_parse(""); Ok(()) } @@ -294,16 +291,16 @@ fn qwen35_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( fn qwen36_answers_without_reasoning_when_thinking_is_disabled( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]", false)?; + let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, &[], false)?; let answer = ChatAnswerGeneration { fixture, - session: &session, + session, sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_ANSWER_TOKENS, } .run(); - answer.assert_content_without_reasoning(&session); + answer.assert_content_without_reasoning(); Ok(()) } @@ -319,16 +316,16 @@ fn qwen36_answers_without_reasoning_when_thinking_is_disabled( fn qwen36_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]", true)?; + let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, &[], true)?; let answer = ChatAnswerGeneration { fixture, - session: &session, + session, sampler: &mut LlamaSampler::greedy()?, max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, } .run(); - answer.assert_reasoning_matches_parse(&session, ""); + answer.assert_reasoning_matches_parse(""); Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs b/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs index e1dec307..cbea2961 100644 --- a/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs +++ b/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs @@ -2,6 +2,7 @@ use anyhow::Context as _; use anyhow::Result; use anyhow::bail; +use llama_cpp_bindings::chat_message_parse_outcome::ChatMessageParseOutcome; use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::error::ingest_error::IngestError; use llama_cpp_bindings::error::parse_chat_message_error::ParseChatMessageError; @@ -19,7 +20,9 @@ use llama_cpp_bindings::sampled_token_section::SampledTokenSection; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings::token::LlamaToken; use llama_cpp_bindings::token_piece::TokenPiece; +use llama_cpp_bindings_tests::ingested_generation::ingested_generation; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; use llama_cpp_fault_injection::armed_fault::ArmedFault; use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; use llama_cpp_test_harness::llama_fixture::LlamaFixture; @@ -58,8 +61,8 @@ use llama_cpp_test_harness_macros::llama_test; n_ubatch = 64, )] fn chat_session_classifier_starts_in_the_pending_section(fixture: &LlamaFixture<'_>) -> Result<()> { - let session = user_turn_chat_session(fixture.model, "hello", "[]", false)?; - let classifier = SampledTokenClassifier::for_chat_session(&session); + let session = user_turn_chat_session(fixture.model, "hello", &[], false)?; + let classifier = SampledTokenClassifier::for_chat_session(fixture.model, session)?; assert_eq!(classifier.current_section(), SampledTokenSection::Pending); Ok(()) @@ -133,8 +136,8 @@ fn undeterminable_classifier_releases_each_token_with_its_visible_and_raw_piece( )] fn finishing_a_chat_answer_settles_its_pending_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { let model = fixture.model; - let session = user_turn_chat_session(model, "hello", "[]", false)?; - let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let session = user_turn_chat_session(model, "hello", &[], false)?; + let mut classifier = SampledTokenClassifier::for_chat_session(fixture.model, session)?; let [ordinary_token] = model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?[..] else { @@ -227,7 +230,7 @@ fn ingesting_a_token_outside_the_vocabulary_reports_it_in_every_classifier( ) -> Result<()> { let model = fixture.model; let token = LlamaToken::new(model.n_vocab()); - let session = user_turn_chat_session(model, "hello", "[]", false)?; + let session = user_turn_chat_session(model, "hello", &[], false)?; let mut outcomes = Vec::new(); let expected = Err(IngestError::Detokenize( TokenToStringError::TokenOutsideVocabulary(TokenOutsideVocabulary { @@ -241,7 +244,8 @@ fn ingesting_a_token_outside_the_vocabulary_reports_it_in_every_classifier( expected ); assert_eq!( - SampledTokenClassifier::for_chat_session(&session).ingest(token, &mut outcomes), + SampledTokenClassifier::for_chat_session(fixture.model, session)? + .ingest(token, &mut outcomes), expected ); assert_eq!(outcomes, []); @@ -266,13 +270,13 @@ fn failed_classification() -> IngestError { fn ingesting_a_generated_token_reports_a_classification_llama_cpp_fails( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let session = user_turn_chat_session(fixture.model, "Hello!", "[]", false)?; - let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let session = user_turn_chat_session(fixture.model, "Hello!", &[], false)?; + let mut classifier = SampledTokenClassifier::for_chat_session(fixture.model, session)?; let generated = fixture .model .str_to_token("Hi", AddBos::Never, ParseSpecialTokens::Never)?; let _fault = - ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_CLASSIFY_THROWS); + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_READ_THROWS); assert_eq!( classifier.ingest(generated[0], &mut Vec::new()), @@ -293,14 +297,14 @@ fn ingesting_a_generated_token_reports_a_classification_llama_cpp_fails( fn ending_the_generation_reports_a_classification_llama_cpp_fails( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let session = user_turn_chat_session(fixture.model, "Hello!", "[]", false)?; - let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let session = user_turn_chat_session(fixture.model, "Hello!", &[], false)?; + let mut classifier = SampledTokenClassifier::for_chat_session(fixture.model, session)?; let end_of_sequence = fixture .model .token_eos() .context("the model declares an EOS token")?; let _fault = - ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_CLASSIFY_THROWS); + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_READ_THROWS); assert_eq!( classifier.ingest(end_of_sequence, &mut Vec::new()), @@ -319,16 +323,17 @@ fn ending_the_generation_reports_a_classification_llama_cpp_fails( n_ubatch = 64, )] fn sampling_reports_a_classification_llama_cpp_fails(fixture: &LlamaFixture<'_>) -> Result<()> { - let session = user_turn_chat_session(fixture.model, "Hello!", "[]", false)?; - let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let session = user_turn_chat_session(fixture.model, "Hello!", &[], false)?; + let prompt_tokens = session.tokenize_prompt(fixture.model)?; + let mut classifier = SampledTokenClassifier::for_chat_session(fixture.model, session)?; let mut context = fixture.build_context()?; let mut batch = LlamaBatch::new(ContextLimits::of(&context)); - classifier.feed_prompt_sequence_to_batch(&mut batch, &session.tokenize_prompt()?, 0, false)?; + classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; context.decode(&batch)?; let _fault = - ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_CLASSIFY_THROWS); + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_READ_THROWS); assert_eq!( classifier @@ -344,3 +349,74 @@ fn sampling_reports_a_classification_llama_cpp_fails(fixture: &LlamaFixture<'_>) Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn an_undeterminable_classifier_has_no_reading_of_the_answer(fixture: &LlamaFixture<'_>) { + let model = fixture.model; + let mut classifier = SampledTokenClassifier::undeterminable(model); + + ingested_generation(&mut classifier, model, "hello"); + + assert!(classifier.latest_reading().is_none()); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn a_chat_answer_classifier_parses_the_answer_it_read_last( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, "hello", &[], false)?; + let mut classifier = SampledTokenClassifier::for_chat_session(fixture.model, session)?; + + ingested_generation(&mut classifier, fixture.model, "Hello there."); + + assert_eq!( + classifier + .latest_reading() + .context("the classifier must have read the answer")? + .parse(), + Ok(&ChatMessageParseOutcome::Recognized( + ParsedChatMessage::new("Hello there.".to_owned(), String::new(), Vec::new()) + )) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn a_chat_answer_classifier_reports_a_failed_reading_of_its_empty_answer( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, "hello", &[], false)?; + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_READ_THROWS); + + assert_eq!( + SampledTokenClassifier::for_chat_session(fixture.model, session).err(), + Some(ParseChatMessageError::Reported { + message: injected_fault_message(), + }) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/token_evaluation_accounting.rs b/llama-cpp-bindings-tests/tests/token_evaluation_accounting.rs new file mode 100644 index 00000000..07596912 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/token_evaluation_accounting.rs @@ -0,0 +1,112 @@ +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + +use anyhow::Result; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +fn fixture_context<'model>(fixture: &LlamaFixture<'model>) -> Result> { + Ok(LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?) +} + +fn greeting_batch(fixture: &LlamaFixture<'_>, context: &LlamaContext<'_>) -> Result { + let tokens = + fixture + .model + .str_to_token("hello there", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(context)); + + batch.add_sequence(&tokens, 0, false)?; + + Ok(batch) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, +)] +fn a_decoded_batch_counts_every_token_llama_cpp_evaluated( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture_context(fixture)?; + let batch = greeting_batch(fixture, &context)?; + + context.decode(&batch)?; + + assert_eq!( + context.evaluated_tokens(), + u64::from(batch.n_tokens().unsigned_abs()) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, +)] +fn an_aborted_decode_counts_no_evaluated_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture_context(fixture)?; + let batch = greeting_batch(fixture, &context)?; + + context.set_abort_flag(Arc::new(AtomicBool::new(true))); + + assert!(context.decode(&batch).is_err()); + assert_eq!(context.evaluated_tokens(), 0); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn evaluated_media_chunks_count_every_token_the_mtmd_helper_decoded( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let bitmap = MtmdBitmap::from_image_data(64, 64, &[128_u8; 64 * 64 * 3])?; + let chunks = mtmd_ctx.tokenize( + MtmdInputText { + text: "Describe: <__media__>".to_owned(), + add_special: true, + parse_special: true, + }, + &[&bitmap], + )?; + let mut context = fixture_context(fixture)?; + let n_batch = i32::try_from(context.n_batch())?; + + chunks.eval_chunks(mtmd_ctx, &mut context, 0, 0, n_batch, true)?; + + assert_eq!(context.evaluated_tokens(), chunks.total_tokens() as u64); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/tool_call_classification.rs b/llama-cpp-bindings-tests/tests/tool_call_classification.rs index b9965ff1..39c2e737 100644 --- a/llama-cpp-bindings-tests/tests/tool_call_classification.rs +++ b/llama-cpp-bindings-tests/tests/tool_call_classification.rs @@ -1,12 +1,11 @@ +use anyhow::Context as _; use anyhow::Result; -use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; use llama_cpp_bindings::ingest_outcome::IngestOutcome; use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; -use llama_cpp_bindings_tests::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; +use llama_cpp_bindings_tests::get_weather_tools::get_weather_tools; use llama_cpp_bindings_tests::ingested_generation::ingested_generation; -use llama_cpp_bindings_tests::recognized_answer::recognized_answer; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -42,13 +41,20 @@ fn qwen35_reasoning_closed_by_a_tool_call_is_attributed_to_each_section( let session = user_turn_chat_session( fixture.model, "What is the weather in Paris?", - GET_WEATHER_TOOLS_JSON, + &get_weather_tools(), true, )?; let answer = format!("{REASONING_BEFORE_THE_CALL}{QWEN_XML_PARIS_WEATHER_CALL}"); - let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let mut classifier = SampledTokenClassifier::for_chat_session(fixture.model, session)?; let outcomes = ingested_generation(&mut classifier, fixture.model, &answer); - let parsed = recognized_answer(&session, &answer, ChatOutputCompleteness::Finished); + let parsed = classifier + .latest_reading() + .context("the classifier must have read the answer")? + .parse() + .cloned() + .map_err(Clone::clone)? + .into_recognized() + .expect("the answer must be recognized"); assert_eq!( raw_text_where(&outcomes, |token| matches!( diff --git a/llama-cpp-bindings-tests/tests/tool_call_grammar.rs b/llama-cpp-bindings-tests/tests/tool_call_grammar.rs index 62b31e3d..3c7b9d8b 100644 --- a/llama-cpp-bindings-tests/tests/tool_call_grammar.rs +++ b/llama-cpp-bindings-tests/tests/tool_call_grammar.rs @@ -1,15 +1,14 @@ use anyhow::Context as _; use anyhow::Result; -use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::chat_answer_generation::ChatAnswerGeneration; use llama_cpp_bindings_tests::paris_weather_tool_call::paris_weather_tool_call; -use llama_cpp_bindings_tests::recognized_answer::recognized_answer; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; use llama_cpp_bindings_tests::weather_chat_session::weather_chat_session; +use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; @@ -17,10 +16,10 @@ const MAX_TOOL_CALL_TOKENS: i32 = 128; fn generated_tool_call_answer( fixture: &LlamaFixture<'_>, - session: &ChatSession<'_>, -) -> Result { + session: ChatSession, +) -> Result { let grammar_sampler = session - .grammar_sampler()? + .grammar_sampler(fixture.model)? .context("a tool request must constrain generation with a grammar")?; let mut sampler = LlamaSampler::chain([grammar_sampler, LlamaSampler::greedy()?])?; let answer = ChatAnswerGeneration { @@ -31,7 +30,10 @@ fn generated_tool_call_answer( } .run(); - Ok(answer.outcome.generated.text().to_owned()) + Ok(answer + .parse + .into_recognized() + .expect("the generated answer must be recognized")) } #[llama_test( @@ -43,9 +45,9 @@ fn generated_tool_call_answer( n_ubatch = 128, )] fn a_chat_session_without_tools_needs_no_grammar_sampler(fixture: &LlamaFixture<'_>) -> Result<()> { - let session = user_turn_chat_session(fixture.model, "Hello!", "[]", false)?; + let session = user_turn_chat_session(fixture.model, "Hello!", &[], false)?; - assert!(session.grammar_sampler()?.is_none()); + assert!(session.grammar_sampler(fixture.model)?.is_none()); Ok(()) } @@ -61,7 +63,7 @@ fn a_chat_session_without_tools_needs_no_grammar_sampler(fixture: &LlamaFixture< fn a_tool_choice_of_none_needs_no_grammar_sampler(fixture: &LlamaFixture<'_>) -> Result<()> { let session = weather_chat_session(fixture.model, ChatToolChoice::None); - assert!(session.grammar_sampler()?.is_none()); + assert!(session.grammar_sampler(fixture.model)?.is_none()); Ok(()) } @@ -78,11 +80,10 @@ fn qwen36_required_tool_call_grammar_constrains_the_answer_to_a_tool_call( fixture: &LlamaFixture<'_>, ) -> Result<()> { let session = weather_chat_session(fixture.model, ChatToolChoice::Required); - let answer = generated_tool_call_answer(fixture, &session)?; assert!(!session.grammar_is_lazy()); assert_eq!( - recognized_answer(&session, &answer, ChatOutputCompleteness::Finished).tool_calls, + generated_tool_call_answer(fixture, session)?.tool_calls, [paris_weather_tool_call()] ); @@ -101,11 +102,10 @@ fn qwen36_automatic_tool_call_grammar_constrains_the_tool_call_after_its_trigger fixture: &LlamaFixture<'_>, ) -> Result<()> { let session = weather_chat_session(fixture.model, ChatToolChoice::Auto); - let answer = generated_tool_call_answer(fixture, &session)?; assert!(session.grammar_is_lazy()); assert_eq!( - recognized_answer(&session, &answer, ChatOutputCompleteness::Finished).tool_calls, + generated_tool_call_answer(fixture, session)?.tool_calls, [paris_weather_tool_call()] ); diff --git a/llama-cpp-bindings-tests/tests/tool_call_parsing.rs b/llama-cpp-bindings-tests/tests/tool_call_parsing.rs index cac715f2..2f62d487 100644 --- a/llama-cpp-bindings-tests/tests/tool_call_parsing.rs +++ b/llama-cpp-bindings-tests/tests/tool_call_parsing.rs @@ -2,15 +2,19 @@ use anyhow::Result; use serde_json::json; use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_message_parse_outcome::ChatMessageParseOutcome; +use llama_cpp_bindings::chat_output::ChatOutput; use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; use llama_cpp_bindings::chat_session::ChatSession; use llama_cpp_bindings::chat_session_options::ChatSessionOptions; use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; -use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; -use llama_cpp_bindings_tests::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; +use llama_cpp_bindings::raw_chat_message::RawChatMessage; +use llama_cpp_bindings_tests::get_weather_tools::get_weather_tools; use llama_cpp_bindings_tests::paris_weather_tool_call::paris_weather_tool_call; use llama_cpp_bindings_tests::recognized_answer::recognized_answer; +use llama_cpp_bindings_tests::user_message::user_message; use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_bindings_types::chat_tool::ChatTool; use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; @@ -18,40 +22,9 @@ use llama_cpp_test_harness::llama_fixture::LlamaFixture; use llama_cpp_test_harness_macros::llama_test; const WEATHER_QUESTION: &str = "What is the weather in Paris?"; +const MALFORMED_QWEN_XML_TOOL_CALL: &str = "garbage"; const PLAIN_CONTENT_REFUSAL: &str = "Sorry, I cannot help with that."; -const NEGOTIATE_WITH_CAT_TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "negotiate_with_cat", - "description": "Attempt to negotiate with a cat. Outcomes are not guaranteed and may include the silent treatment.", - "parameters": { - "type": "object", - "properties": { - "topic": { - "type": "string", - "description": "What you are trying to negotiate, e.g. 'get off the keyboard' or 'stop knocking things off the table'" - }, - "bribe": { - "type": "string", - "enum": ["tuna", "salmon", "treats", "ear_scritches", "cardboard_box", "none"], - "description": "What you are offering in exchange" - }, - "desperation_level": { - "type": "integer", - "description": "How desperate you are, on a scale from 1 (mildly annoyed human) to 10 (it is 3am)", - "minimum": 1, - "maximum": 10 - } - }, - "required": ["topic"], - "additionalProperties": false - } - } - } -]"#; - const QWEN_XML_PARIS_WEATHER_CALL: &str = "\n\ \n\ \n\ @@ -60,13 +33,38 @@ Paris\n\ \n\ "; +fn negotiate_with_cat_tools() -> Vec { + vec![ChatTool { + name: "negotiate_with_cat".to_owned(), + description: "Attempt to negotiate with a cat. Outcomes are not guaranteed and may include the silent treatment.".to_owned(), + parameters: Some(json!({ + "type": "object", + "properties": { + "topic": { + "type": "string", + "description": "What you are trying to negotiate, e.g. 'get off the keyboard' or 'stop knocking things off the table'" + }, + "bribe": { + "type": "string", + "enum": ["tuna", "salmon", "treats", "ear_scritches", "cardboard_box", "none"], + "description": "What you are offering in exchange" + }, + "desperation_level": { + "type": "integer", + "description": "How desperate you are, on a scale from 1 (mildly annoyed human) to 10 (it is 3am)", + "minimum": 1, + "maximum": 10 + } + }, + "required": ["topic"], + "additionalProperties": false + })), + }] +} + fn assert_parses_to_paris_weather_call(fixture: &LlamaFixture<'_>, answer: &str) -> Result<()> { - let session = user_turn_chat_session( - fixture.model, - WEATHER_QUESTION, - GET_WEATHER_TOOLS_JSON, - false, - )?; + let session = + user_turn_chat_session(fixture.model, WEATHER_QUESTION, &get_weather_tools(), false)?; let parsed = recognized_answer(&session, answer, ChatOutputCompleteness::Finished); assert_eq!(parsed.tool_calls, [paris_weather_tool_call()]); @@ -145,7 +143,7 @@ fn qwen35_parses_tool_call_arguments_into_their_schema_types( let session = user_turn_chat_session( fixture.model, "Get the cat off the keyboard.", - NEGOTIATE_WITH_CAT_TOOLS_JSON, + &negotiate_with_cat_tools(), false, )?; let parsed = recognized_answer( @@ -193,12 +191,8 @@ tuna\n\ fn qwen35_parses_a_tool_call_cut_inside_its_arguments_as_a_named_call( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let session = user_turn_chat_session( - fixture.model, - WEATHER_QUESTION, - GET_WEATHER_TOOLS_JSON, - false, - )?; + let session = + user_turn_chat_session(fixture.model, WEATHER_QUESTION, &get_weather_tools(), false)?; let parsed = recognized_answer( &session, "\n\n) -> let session = ChatSession::new( fixture.model, fixture.model.chat_templates()?, - &[LlamaChatMessage::new( - "user".to_owned(), - "What is the weather in Paris and in Berlin?".to_owned(), - )?], - GET_WEATHER_TOOLS_JSON, + &[user_message("What is the weather in Paris and in Berlin?")], + &get_weather_tools(), &ChatSessionOptions { assistant_turn: ChatAssistantTurn::New, enable_thinking: false, @@ -284,12 +275,8 @@ fn qwen35_parses_consecutive_tool_calls_in_order(fixture: &LlamaFixture<'_>) -> fn plain_content_answering_a_tool_request_parses_without_tool_calls( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let session = user_turn_chat_session( - fixture.model, - WEATHER_QUESTION, - GET_WEATHER_TOOLS_JSON, - false, - )?; + let session = + user_turn_chat_session(fixture.model, WEATHER_QUESTION, &get_weather_tools(), false)?; let parsed = recognized_answer( &session, PLAIN_CONTENT_REFUSAL, @@ -312,10 +299,41 @@ fn plain_content_answering_a_tool_request_parses_without_tool_calls( fn deepseek_r1_8b_plain_content_without_tools_parses_without_tool_calls( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let session = user_turn_chat_session(fixture.model, "Hello!", "[]", false)?; + let session = user_turn_chat_session(fixture.model, "Hello!", &[], false)?; let parsed = recognized_answer(&session, "Hello there.", ChatOutputCompleteness::Finished); assert_eq!(parsed.tool_calls, []); Ok(()) } + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_reports_a_finished_tool_call_its_parser_stops_short_of_with_the_format_it_expected( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = + user_turn_chat_session(fixture.model, WEATHER_QUESTION, &get_weather_tools(), false)?; + + assert_eq!( + session + .read( + &ChatOutput::from(MALFORMED_QWEN_XML_TOOL_CALL), + ChatOutputCompleteness::Finished + )? + .parse(), + Ok(&ChatMessageParseOutcome::Unrecognized(RawChatMessage { + text: MALFORMED_QWEN_XML_TOOL_CALL.to_owned(), + completeness: ChatOutputCompleteness::Finished, + expected_format: "peg-native".to_owned(), + })) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-types/src/chat_message.rs b/llama-cpp-bindings-types/src/chat_message.rs new file mode 100644 index 00000000..31b1c7a4 --- /dev/null +++ b/llama-cpp-bindings-types/src/chat_message.rs @@ -0,0 +1,163 @@ +use serde_json::Map; +use serde_json::Value; + +use crate::chat_message_content::ChatMessageContent; +use crate::chat_message_tool_call::ChatMessageToolCall; + +const ASSISTANT_ROLE: &str = "assistant"; +const TOOL_ROLE: &str = "tool"; + +fn message_of(role: &str, content: &ChatMessageContent) -> Map { + Map::from_iter([ + ("role".to_owned(), Value::String(role.to_owned())), + ("content".to_owned(), Value::from(content)), + ]) +} + +fn with_reasoning( + mut message: Map, + reasoning_content: Option<&String>, +) -> Map { + if let Some(reasoning_content) = reasoning_content { + message.insert( + "reasoning_content".to_owned(), + Value::String(reasoning_content.clone()), + ); + } + + message +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ChatMessage { + Message { + role: String, + content: ChatMessageContent, + reasoning_content: Option, + }, + AssistantToolCalls { + content: ChatMessageContent, + reasoning_content: Option, + tool_calls: Vec, + }, + ToolResult { + content: ChatMessageContent, + tool_call_id: String, + }, +} + +impl From<&ChatMessage> for Value { + fn from(message: &ChatMessage) -> Self { + Self::Object(match message { + ChatMessage::Message { + role, + content, + reasoning_content, + } => with_reasoning(message_of(role, content), reasoning_content.as_ref()), + ChatMessage::AssistantToolCalls { + content, + reasoning_content, + tool_calls, + } => { + let mut assistant_turn = with_reasoning( + message_of(ASSISTANT_ROLE, content), + reasoning_content.as_ref(), + ); + + assistant_turn.insert( + "tool_calls".to_owned(), + tool_calls.iter().map(Self::from).collect(), + ); + + assistant_turn + } + ChatMessage::ToolResult { + content, + tool_call_id, + } => { + let mut tool_result = message_of(TOOL_ROLE, content); + + tool_result.insert( + "tool_call_id".to_owned(), + Self::String(tool_call_id.clone()), + ); + + tool_result + } + }) + } +} + +#[cfg(test)] +mod tests { + use serde_json::Value; + use serde_json::json; + + use super::ChatMessage; + use crate::chat_message_content::ChatMessageContent; + use crate::chat_message_tool_call::ChatMessageToolCall; + + #[test] + fn a_message_carries_its_own_role() { + assert_eq!( + Value::from(&ChatMessage::Message { + role: "user".to_owned(), + content: ChatMessageContent::Text("Hello".to_owned()), + reasoning_content: None, + }), + json!({"role": "user", "content": "Hello"}) + ); + } + + #[test] + fn a_reasoned_message_carries_its_reasoning() { + assert_eq!( + Value::from(&ChatMessage::Message { + role: "assistant".to_owned(), + content: ChatMessageContent::Text("Paris".to_owned()), + reasoning_content: Some("The capital of France".to_owned()), + }), + json!({ + "role": "assistant", + "content": "Paris", + "reasoning_content": "The capital of France", + }) + ); + } + + #[test] + fn an_assistant_tool_call_turn_is_an_assistant_message_with_tool_calls() { + assert_eq!( + Value::from(&ChatMessage::AssistantToolCalls { + content: ChatMessageContent::Text(String::new()), + reasoning_content: Some("I need the weather".to_owned()), + tool_calls: vec![ChatMessageToolCall { + id: "call_weather".to_owned(), + name: "get_weather".to_owned(), + arguments: "{}".to_owned(), + }], + }), + json!({ + "role": "assistant", + "content": "", + "reasoning_content": "I need the weather", + "tool_calls": [{ + "id": "call_weather", + "type": "function", + "function": {"name": "get_weather", "arguments": "{}"}, + }], + }) + ); + } + + #[test] + fn a_tool_result_is_a_tool_message_answering_its_call() { + assert_eq!( + Value::from(&ChatMessage::ToolResult { + content: ChatMessageContent::Text("sunny".to_owned()), + tool_call_id: "call_weather".to_owned(), + }), + json!({"role": "tool", "content": "sunny", "tool_call_id": "call_weather"}) + ); + } +} diff --git a/llama-cpp-bindings-types/src/chat_message_content.rs b/llama-cpp-bindings-types/src/chat_message_content.rs new file mode 100644 index 00000000..f1b72237 --- /dev/null +++ b/llama-cpp-bindings-types/src/chat_message_content.rs @@ -0,0 +1,49 @@ +use serde_json::Value; + +use crate::chat_message_content_part::ChatMessageContentPart; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ChatMessageContent { + Text(String), + Parts(Vec), +} + +impl From<&ChatMessageContent> for Value { + fn from(content: &ChatMessageContent) -> Self { + match content { + ChatMessageContent::Text(text) => Self::String(text.clone()), + ChatMessageContent::Parts(parts) => parts.iter().map(Self::from).collect(), + } + } +} + +#[cfg(test)] +mod tests { + use serde_json::Value; + use serde_json::json; + + use super::ChatMessageContent; + use crate::chat_message_content_part::ChatMessageContentPart; + + #[test] + fn text_content_is_a_plain_string() { + assert_eq!( + Value::from(&ChatMessageContent::Text("Hello".to_owned())), + json!("Hello") + ); + } + + #[test] + fn content_parts_are_an_array_of_typed_parts() { + assert_eq!( + Value::from(&ChatMessageContent::Parts(vec![ + ChatMessageContentPart::MediaMarker("<__media__>".to_owned()), + ChatMessageContentPart::Text("Describe".to_owned()), + ])), + json!([ + {"type": "media_marker", "text": "<__media__>"}, + {"type": "text", "text": "Describe"}, + ]) + ); + } +} diff --git a/llama-cpp-bindings-types/src/chat_message_content_part.rs b/llama-cpp-bindings-types/src/chat_message_content_part.rs new file mode 100644 index 00000000..b0bdf270 --- /dev/null +++ b/llama-cpp-bindings-types/src/chat_message_content_part.rs @@ -0,0 +1,45 @@ +use serde_json::Value; +use serde_json::json; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ChatMessageContentPart { + Text(String), + MediaMarker(String), +} + +impl From<&ChatMessageContentPart> for Value { + fn from(part: &ChatMessageContentPart) -> Self { + match part { + ChatMessageContentPart::Text(text) => json!({"type": "text", "text": text}), + ChatMessageContentPart::MediaMarker(marker) => { + json!({"type": "media_marker", "text": marker}) + } + } + } +} + +#[cfg(test)] +mod tests { + use serde_json::Value; + use serde_json::json; + + use super::ChatMessageContentPart; + + #[test] + fn a_text_part_is_typed_as_text() { + assert_eq!( + Value::from(&ChatMessageContentPart::Text("Describe".to_owned())), + json!({"type": "text", "text": "Describe"}) + ); + } + + #[test] + fn a_media_marker_part_is_typed_as_a_media_marker() { + assert_eq!( + Value::from(&ChatMessageContentPart::MediaMarker( + "<__media__>".to_owned() + )), + json!({"type": "media_marker", "text": "<__media__>"}) + ); + } +} diff --git a/llama-cpp-bindings-types/src/chat_message_tool_call.rs b/llama-cpp-bindings-types/src/chat_message_tool_call.rs new file mode 100644 index 00000000..f6e2eb40 --- /dev/null +++ b/llama-cpp-bindings-types/src/chat_message_tool_call.rs @@ -0,0 +1,52 @@ +use serde_json::Value; +use serde_json::json; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ChatMessageToolCall { + pub id: String, + pub name: String, + pub arguments: String, +} + +impl From<&ChatMessageToolCall> for Value { + fn from( + ChatMessageToolCall { + id, + name, + arguments, + }: &ChatMessageToolCall, + ) -> Self { + json!({ + "id": id, + "type": "function", + "function": {"name": name, "arguments": arguments}, + }) + } +} + +#[cfg(test)] +mod tests { + use serde_json::Value; + use serde_json::json; + + use super::ChatMessageToolCall; + + #[test] + fn a_tool_call_is_an_openai_function_call() { + assert_eq!( + Value::from(&ChatMessageToolCall { + id: "call_weather".to_owned(), + name: "get_weather".to_owned(), + arguments: r#"{"location":"Paris"}"#.to_owned(), + }), + json!({ + "id": "call_weather", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"location\":\"Paris\"}", + }, + }) + ); + } +} diff --git a/llama-cpp-bindings-types/src/chat_tool.rs b/llama-cpp-bindings-types/src/chat_tool.rs new file mode 100644 index 00000000..45285263 --- /dev/null +++ b/llama-cpp-bindings-types/src/chat_tool.rs @@ -0,0 +1,78 @@ +use serde_json::Map; +use serde_json::Value; +use serde_json::json; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ChatTool { + pub name: String, + pub description: String, + pub parameters: Option, +} + +impl From<&ChatTool> for Value { + fn from( + ChatTool { + name, + description, + parameters, + }: &ChatTool, + ) -> Self { + let mut function = Map::from_iter([ + ("name".to_owned(), Self::String(name.clone())), + ("description".to_owned(), Self::String(description.clone())), + ]); + + if let Some(parameters) = parameters { + function.insert("parameters".to_owned(), parameters.clone()); + } + + json!({"type": "function", "function": function}) + } +} + +#[cfg(test)] +mod tests { + use serde_json::Value; + use serde_json::json; + + use super::ChatTool; + + #[test] + fn a_tool_with_parameters_is_an_openai_function_with_its_schema() { + let schema = json!({"type": "object", "properties": {"location": {"type": "string"}}}); + + assert_eq!( + Value::from(&ChatTool { + name: "get_weather".to_owned(), + description: "Reports the weather".to_owned(), + parameters: Some(schema.clone()), + }), + json!({ + "type": "function", + "function": { + "name": "get_weather", + "description": "Reports the weather", + "parameters": schema, + }, + }) + ); + } + + #[test] + fn a_tool_without_parameters_is_an_openai_function_without_them() { + assert_eq!( + Value::from(&ChatTool { + name: "get_time".to_owned(), + description: "Reports the time".to_owned(), + parameters: None, + }), + json!({ + "type": "function", + "function": { + "name": "get_time", + "description": "Reports the time", + }, + }) + ); + } +} diff --git a/llama-cpp-bindings-types/src/lib.rs b/llama-cpp-bindings-types/src/lib.rs index f6354c6a..046da4ec 100644 --- a/llama-cpp-bindings-types/src/lib.rs +++ b/llama-cpp-bindings-types/src/lib.rs @@ -3,6 +3,11 @@ deny(clippy::unwrap_used, clippy::expect_used, clippy::panic) )] +pub mod chat_message; +pub mod chat_message_content; +pub mod chat_message_content_part; +pub mod chat_message_tool_call; +pub mod chat_tool; pub mod invalid_arguments_json; pub mod model_provided_tool_call_id; pub mod model_provided_tool_call_id_error; diff --git a/llama-cpp-bindings/src/chat_classification.rs b/llama-cpp-bindings/src/chat_classification.rs index b1cd0b02..965f7aa2 100644 --- a/llama-cpp-bindings/src/chat_classification.rs +++ b/llama-cpp-bindings/src/chat_classification.rs @@ -3,12 +3,13 @@ use std::ops::Range; use crate::chat_output_completeness::ChatOutputCompleteness; use crate::chat_piece_role::ChatPieceRole; use crate::chat_spans::ChatSpans; +use crate::raw_chat_message::RawChatMessage; use crate::sampled_token_section::SampledTokenSection; #[derive(Clone, Debug, Eq, PartialEq)] pub enum ChatClassification { Spans(ChatSpans), - OutputDoesNotMatchFormat(ChatOutputCompleteness), + OutputDoesNotMatchFormat(RawChatMessage), } impl ChatClassification { @@ -19,10 +20,24 @@ impl ChatClassification { pub fn classify_token(&self, bytes: &Range) -> Option { match self { Self::Spans(spans) => spans.classify_token(bytes), - Self::OutputDoesNotMatchFormat(ChatOutputCompleteness::Partial) => None, - Self::OutputDoesNotMatchFormat(ChatOutputCompleteness::Finished) => { - Some(ChatPieceRole::Text(SampledTokenSection::Pending)) - } + Self::OutputDoesNotMatchFormat(RawChatMessage { + completeness: ChatOutputCompleteness::Partial, + .. + }) => None, + Self::OutputDoesNotMatchFormat(RawChatMessage { + completeness: ChatOutputCompleteness::Finished, + .. + }) => Some(ChatPieceRole::Text(SampledTokenSection::Pending)), + } + } + + /// The section the output ends in; an answer that does not match the template's format + /// ends in no section. + #[must_use] + pub fn trailing_section(&self) -> SampledTokenSection { + match self { + Self::Spans(spans) => spans.trailing_section(), + Self::OutputDoesNotMatchFormat(_) => SampledTokenSection::Pending, } } } @@ -32,13 +47,24 @@ mod tests { use crate::chat_classification::ChatClassification; use crate::chat_output_completeness::ChatOutputCompleteness; use crate::chat_piece_role::ChatPieceRole; + use crate::chat_span::ChatSpan; + use crate::chat_span_kind::ChatSpanKind; + use crate::chat_spans::ChatSpans; + use crate::raw_chat_message::RawChatMessage; use crate::sampled_token_section::SampledTokenSection; + fn unrecognized(completeness: ChatOutputCompleteness) -> ChatClassification { + ChatClassification::OutputDoesNotMatchFormat(RawChatMessage { + text: "odd".to_owned(), + completeness, + expected_format: "peg-native".to_owned(), + }) + } + #[test] fn a_partial_answer_that_does_not_match_the_format_settles_no_token() { assert_eq!( - ChatClassification::OutputDoesNotMatchFormat(ChatOutputCompleteness::Partial) - .classify_token(&(0..3)), + unrecognized(ChatOutputCompleteness::Partial).classify_token(&(0..3)), None ); } @@ -46,9 +72,32 @@ mod tests { #[test] fn a_finished_answer_that_does_not_match_the_format_is_visible_text_of_no_section() { assert_eq!( - ChatClassification::OutputDoesNotMatchFormat(ChatOutputCompleteness::Finished) - .classify_token(&(0..3)), + unrecognized(ChatOutputCompleteness::Finished).classify_token(&(0..3)), Some(ChatPieceRole::Text(SampledTokenSection::Pending)) ); } + + #[test] + fn an_answer_that_matches_the_format_ends_in_the_section_of_its_last_span() { + assert_eq!( + ChatClassification::Spans(ChatSpans::new( + 4, + &[ChatSpan { + kind: ChatSpanKind::ToolCall, + bytes: 0..4, + }], + ChatOutputCompleteness::Partial, + )) + .trailing_section(), + SampledTokenSection::ToolCall + ); + } + + #[test] + fn an_answer_that_does_not_match_the_format_ends_in_no_section() { + assert_eq!( + unrecognized(ChatOutputCompleteness::Finished).trailing_section(), + SampledTokenSection::Pending + ); + } } diff --git a/llama-cpp-bindings/src/chat_message_parse_outcome.rs b/llama-cpp-bindings/src/chat_message_parse_outcome.rs index 10c277e2..f2620895 100644 --- a/llama-cpp-bindings/src/chat_message_parse_outcome.rs +++ b/llama-cpp-bindings/src/chat_message_parse_outcome.rs @@ -2,7 +2,7 @@ use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; use crate::raw_chat_message::RawChatMessage; -#[derive(Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq)] pub enum ChatMessageParseOutcome { Recognized(ParsedChatMessage), Unrecognized(RawChatMessage), @@ -34,7 +34,7 @@ mod tests { RawChatMessage { text: "Paris.".to_owned(), completeness: ChatOutputCompleteness::Finished, - ffi_error_message: "the format differs".to_owned(), + expected_format: "peg-native".to_owned(), } } diff --git a/llama-cpp-bindings/src/chat_reading.rs b/llama-cpp-bindings/src/chat_reading.rs new file mode 100644 index 00000000..a6719912 --- /dev/null +++ b/llama-cpp-bindings/src/chat_reading.rs @@ -0,0 +1,294 @@ +use std::cell::OnceCell; +use std::ffi::c_char; +use std::ptr; +use std::ptr::NonNull; +use std::slice; + +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; + +use crate::chat_classification::ChatClassification; +use crate::chat_message_parse_outcome::ChatMessageParseOutcome; +use crate::chat_output::ChatOutput; +use crate::chat_output_completeness::ChatOutputCompleteness; +use crate::chat_span::ChatSpan; +use crate::chat_span_kind::ChatSpanKind; +use crate::chat_spans::ChatSpans; +use crate::cpp_text_view::cpp_text_view; +use crate::error::parse_chat_message_error::ParseChatMessageError; +use crate::parsed_chat::take_parsed_chat_message; +use crate::raw_chat_message::RawChatMessage; + +const fn contract_violation(operation: &'static str, detail: &'static str) -> FfiContractError { + FfiContractError { operation, detail } +} + +fn spans_of( + reading: NonNull, + completeness: ChatOutputCompleteness, +) -> Result { + let view = + unsafe { llama_cpp_bindings_sys::llama_rs_chat_reading_spans_view(reading.as_ptr()) }; + let wrapper_spans = NonNull::new(view.spans.cast_mut()).map_or(&[][..], |first_span| unsafe { + slice::from_raw_parts(first_span.as_ptr(), view.spans_count) + }); + + wrapper_spans + .iter() + .map(|wrapper_span| { + ChatSpanKind::try_from(wrapper_span.kind).map(|kind| ChatSpan { + kind, + bytes: wrapper_span.start..wrapper_span.end, + }) + }) + .collect::, FfiStatusError>>() + .map(|spans| ChatSpans::new(view.parsed_length, &spans, completeness)) + .map_err(ParseChatMessageError::from) +} + +fn expected_format_of( + reading: NonNull, +) -> Result { + let mut length = 0; + let format_name = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_reading_format_name(reading.as_ptr(), &raw mut length) + }; + + unsafe { cpp_text_view(format_name, length) } + .map(str::to_owned) + .map_err(ParseChatMessageError::FormatNameNotUtf8) +} + +fn classification_of( + reading: NonNull, + output: &ChatOutput, + completeness: ChatOutputCompleteness, +) -> Result { + if unsafe { llama_cpp_bindings_sys::llama_rs_chat_reading_matches_format(reading.as_ptr()) } { + return spans_of(reading, completeness).map(ChatClassification::Spans); + } + + Ok(ChatClassification::OutputDoesNotMatchFormat( + RawChatMessage { + text: output.text().to_owned(), + completeness, + expected_format: expected_format_of(reading)?, + }, + )) +} + +fn chat_reading_parse_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_chat_reading_parse_status, + out_handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, + out_error: *mut c_char, +) -> Result, ParseChatMessageError> { + const OPERATION: &str = "llama_rs_chat_reading_parse"; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_READING_PARSE_OK => NonNull::new(out_handle) + .ok_or_else(|| { + contract_violation(OPERATION, "success status contained a null handle").into() + }), + llama_cpp_bindings_sys::LLAMA_RS_CHAT_READING_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION => { + Err(ParseChatMessageError::Reported { + message: unsafe { + read_and_free_cpp_string( + out_error, + OPERATION, + "reported a thrown C++ exception without an error message", + ) + }?, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_READING_PARSE_ERROR_STRING_ALLOCATION_FAILED => { + Err(ParseChatMessageError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_READING_PARSE_LLAMA_CPP_OUT_OF_MEMORY => { + Err(ParseChatMessageError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_READING_PARSE_NULL_READING_ARG => { + Err(contract_violation(OPERATION, "was given a null reading argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_READING_PARSE_NULL_OUT_HANDLE_ARG => { + Err(contract_violation(OPERATION, "was given a null out_handle argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_READING_PARSE_NULL_OUT_ERROR_ARG => { + Err(contract_violation(OPERATION, "was given a null out_error argument").into()) + } + other => Err(FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), + } +} + +fn parsed_message_of( + reading: NonNull, +) -> Result { + let mut out_handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_reading_parse( + reading.as_ptr(), + &raw mut out_handle, + &raw mut out_error, + ) + }; + + chat_reading_parse_status_to_result(status, out_handle, out_error) + .and_then(take_parsed_chat_message) + .map(ChatMessageParseOutcome::Recognized) +} + +/// One reading of the model's answer by the template's parser. +/// +/// It classifies the answer's bytes right away and builds the parsed message the first time it +/// is asked for: a partial answer's message comes from the same parse, and a finished answer's +/// message comes from llama.cpp completing it as llama.cpp completes every finished message. +pub struct ChatReading { + reading: NonNull, + classification: ChatClassification, + parse_outcome: OnceCell>, +} + +unsafe impl Send for ChatReading {} + +impl ChatReading { + /// Takes ownership of `reading`, the parse of `output`, and classifies it. + /// + /// # Errors + /// + /// Returns [`ParseChatMessageError`] when a span kind is unknown to this binding or the + /// expected format's name is not UTF-8; the reading is released either way. + pub fn take( + reading: NonNull, + output: &ChatOutput, + completeness: ChatOutputCompleteness, + ) -> Result { + match classification_of(reading, output, completeness) { + Ok(classification) => Ok(Self { + reading, + classification, + parse_outcome: OnceCell::new(), + }), + Err(classification_failure) => { + unsafe { llama_cpp_bindings_sys::llama_rs_chat_reading_free(reading.as_ptr()) }; + + Err(classification_failure) + } + } + } + + #[must_use] + pub const fn classification(&self) -> &ChatClassification { + &self.classification + } + + /// The message this reading holds, or the raw answer the parser did not recognize, built + /// once and kept for every later call. + /// + /// # Errors + /// + /// Returns the [`ParseChatMessageError`] of llama.cpp failing while mapping the parse into a + /// message, or reporting text that is not UTF-8. + pub fn parse(&self) -> Result<&ChatMessageParseOutcome, &ParseChatMessageError> { + self.parse_outcome + .get_or_init(|| match &self.classification { + ChatClassification::Spans(_) => parsed_message_of(self.reading), + ChatClassification::OutputDoesNotMatchFormat(raw_chat_message) => Ok( + ChatMessageParseOutcome::Unrecognized(raw_chat_message.clone()), + ), + }) + .as_ref() + } +} + +impl Drop for ChatReading { + fn drop(&mut self) { + unsafe { llama_cpp_bindings_sys::llama_rs_chat_reading_free(self.reading.as_ptr()) }; + } +} + +#[cfg(test)] +mod tests { + use std::ptr; + + use llama_cpp_ffi_status::cpp_string_error::CppStringError; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + + use super::chat_reading_parse_status_to_result; + use crate::error::parse_chat_message_error::ParseChatMessageError; + + #[test] + fn parsing_a_reading_maps_every_failure_status() { + let parse = |status, out_error| { + chat_reading_parse_status_to_result(status, ptr::null_mut(), out_error).err() + }; + let contract = |detail| { + Some(ParseChatMessageError::FfiContract(FfiContractError { + operation: "llama_rs_chat_reading_parse", + detail, + })) + }; + + assert_eq!( + [ + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_READING_PARSE_OK, + ptr::null_mut() + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_READING_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION, + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"the mapper threw".as_ptr()) } + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_READING_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut() + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_READING_PARSE_ERROR_STRING_ALLOCATION_FAILED, + ptr::null_mut() + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_READING_PARSE_LLAMA_CPP_OUT_OF_MEMORY, + ptr::null_mut() + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_READING_PARSE_NULL_READING_ARG, + ptr::null_mut() + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_READING_PARSE_NULL_OUT_HANDLE_ARG, + ptr::null_mut() + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_READING_PARSE_NULL_OUT_ERROR_ARG, + ptr::null_mut() + ), + parse(255, ptr::null_mut()), + ], + [ + contract("success status contained a null handle"), + Some(ParseChatMessageError::Reported { + message: "the mapper threw".to_owned() + }), + Some(ParseChatMessageError::CppString(CppStringError::Missing(FfiContractError { + operation: "llama_rs_chat_reading_parse", + detail: "reported a thrown C++ exception without an error message", + }))), + Some(ParseChatMessageError::NotEnoughMemory), + Some(ParseChatMessageError::LlamaCppOutOfMemory), + contract("was given a null reading argument"), + contract("was given a null out_handle argument"), + contract("was given a null out_error argument"), + Some(ParseChatMessageError::FfiStatus(FfiStatusError { + operation: "llama_rs_chat_reading_parse", + code: 255 + })), + ] + ); + } +} diff --git a/llama-cpp-bindings/src/chat_session.rs b/llama-cpp-bindings/src/chat_session.rs index ed9a359c..cfa5d482 100644 --- a/llama-cpp-bindings/src/chat_session.rs +++ b/llama-cpp-bindings/src/chat_session.rs @@ -1,28 +1,23 @@ -use std::ffi::CString; use std::ffi::c_char; use std::ptr; use std::ptr::NonNull; -use std::slice; - -use serde_json::Value; -use serde_json::json; +use llama_cpp_bindings_types::chat_message::ChatMessage; +use llama_cpp_bindings_types::chat_message_content::ChatMessageContent; +use llama_cpp_bindings_types::chat_tool::ChatTool; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; +use serde_json::Value; use crate::chat_assistant_prefill::ChatAssistantPrefill; use crate::chat_assistant_turn::ChatAssistantTurn; -use crate::chat_classification::ChatClassification; use crate::chat_grammar_trigger::ChatGrammarTrigger; -use crate::chat_message_parse_outcome::ChatMessageParseOutcome; use crate::chat_output::ChatOutput; use crate::chat_output_completeness::ChatOutputCompleteness; use crate::chat_prompt_part::ChatPromptPart; +use crate::chat_reading::ChatReading; use crate::chat_session_options::ChatSessionOptions; -use crate::chat_span::ChatSpan; -use crate::chat_span_kind::ChatSpanKind; -use crate::chat_spans::ChatSpans; use crate::chat_templates::ChatTemplates; use crate::cpp_text_view::cpp_text_view; use crate::error::chat_session_error::ChatSessionError; @@ -30,53 +25,43 @@ use crate::error::parse_chat_message_error::ParseChatMessageError; use crate::lazy_grammar_triggers::LazyGrammarTriggers; use crate::model::LlamaModel; use crate::model::add_bos::AddBos; -use crate::model::llama_chat_message::LlamaChatMessage; use crate::model::parse_special_tokens::ParseSpecialTokens; -use crate::parsed_chat::take_parsed_chat_message; -use crate::raw_chat_message::RawChatMessage; use crate::sampling::LlamaSampler; use crate::token::LlamaToken; +const ASSISTANT_ROLE: &str = "assistant"; const GRAMMAR_ROOT: &str = "root"; -fn chat_message_json(message: &LlamaChatMessage) -> Result { - Ok(json!({ - "role": message.role.to_str().map_err(ChatSessionError::MessageNotUtf8)?, - "content": message.content.to_str().map_err(ChatSessionError::MessageNotUtf8)?, - })) -} - -fn continued_assistant_message_json(assistant_turn: &ChatAssistantTurn) -> Option { +fn continued_assistant_message(assistant_turn: &ChatAssistantTurn) -> Option { match assistant_turn { ChatAssistantTurn::Omitted | ChatAssistantTurn::New => None, - ChatAssistantTurn::ContinueReasoning(reasoning_content) => Some(json!({ - "role": "assistant", - "content": "", - "reasoning_content": reasoning_content, - })), + ChatAssistantTurn::ContinueReasoning(reasoning_content) => Some(ChatMessage::Message { + role: ASSISTANT_ROLE.to_owned(), + content: ChatMessageContent::Text(String::new()), + reasoning_content: Some(reasoning_content.clone()), + }), ChatAssistantTurn::ContinueContent(ChatAssistantPrefill { reasoning_content, content, - }) => Some(json!({ - "role": "assistant", - "content": content, - "reasoning_content": reasoning_content, - })), + }) => Some(ChatMessage::Message { + role: ASSISTANT_ROLE.to_owned(), + content: ChatMessageContent::Text(content.clone()), + reasoning_content: Some(reasoning_content.clone()), + }), } } -fn messages_json( - messages: &[LlamaChatMessage], - assistant_turn: &ChatAssistantTurn, -) -> Result { - let mut messages = messages +fn messages_json(messages: &[ChatMessage], assistant_turn: &ChatAssistantTurn) -> String { + messages .iter() - .map(chat_message_json) - .collect::, ChatSessionError>>()?; - - messages.extend(continued_assistant_message_json(assistant_turn)); + .chain(continued_assistant_message(assistant_turn).as_ref()) + .map(Value::from) + .collect::() + .to_string() +} - CString::new(Value::Array(messages).to_string()).map_err(ChatSessionError::RequestContainsNul) +fn tools_json(tools: &[ChatTool]) -> String { + tools.iter().map(Value::from).collect::().to_string() } fn session_text(text: *const c_char, length: usize) -> Result { @@ -111,16 +96,6 @@ fn chat_session_create_status_to_result( .ok_or_else(|| { contract_violation(OPERATION, "success status contained a null session").into() }), - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_NOT_JSON => { - Err(ChatSessionError::ToolsNotJson { - message: reported_message()?, - }) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_INVALID => { - Err(ChatSessionError::ToolsInvalid { - message: reported_message()?, - }) - } llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TEMPLATE_CANNOT_SERVE_REQUEST => { Err(ChatSessionError::TemplateCannotServeRequest { message: reported_message()?, @@ -163,114 +138,19 @@ fn chat_session_create_status_to_result( } } -fn chat_session_parse_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_chat_session_parse_status, - out_handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, +fn chat_session_read_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_chat_session_read_status, + out_reading: llama_cpp_bindings_sys::llama_rs_chat_reading_handle, out_error: *mut c_char, - output: &ChatOutput, - completeness: ChatOutputCompleteness, -) -> Result { - const OPERATION: &str = "llama_rs_chat_session_parse"; - let reported_message = || unsafe { - read_and_free_cpp_string( - out_error, - OPERATION, - "reported a failure without an error message", - ) - }; +) -> Result, ParseChatMessageError> { + const OPERATION: &str = "llama_rs_chat_session_read"; match status { - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_OK => NonNull::new(out_handle) + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_READ_OK => NonNull::new(out_reading) .ok_or_else(|| { - contract_violation(OPERATION, "success status contained a null handle").into() - }) - .and_then(take_parsed_chat_message) - .map(ChatMessageParseOutcome::Recognized), - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_OUTPUT_DOES_NOT_MATCH_FORMAT => { - Ok(ChatMessageParseOutcome::Unrecognized(RawChatMessage { - text: output.text().to_owned(), - completeness, - ffi_error_message: reported_message()?, - })) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION => { - Err(ParseChatMessageError::Reported { - message: reported_message()?, - }) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_ERROR_STRING_ALLOCATION_FAILED => { - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_SESSION_ARG => { - Err(contract_violation(OPERATION, "was given a null session argument").into()) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_TEXT_ARG => { - Err(contract_violation(OPERATION, "was given a null text argument").into()) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_BYTE_TOKENS_ARG => { - Err(contract_violation(OPERATION, "was given a null byte_tokens argument").into()) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_HANDLE_ARG => { - Err(contract_violation(OPERATION, "was given a null out_handle argument").into()) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_ERROR_ARG => { - Err(contract_violation(OPERATION, "was given a null out_error argument").into()) - } - other => Err(FfiStatusError { - operation: OPERATION, - code: i64::from(other), - } - .into()), - } -} - -fn read_chat_spans( - spans: NonNull, - completeness: ChatOutputCompleteness, -) -> Result { - let view = unsafe { llama_cpp_bindings_sys::llama_rs_chat_spans_view_of(spans.as_ptr()) }; - let wrapper_spans = NonNull::new(view.spans.cast_mut()).map_or(&[][..], |first_span| unsafe { - slice::from_raw_parts(first_span.as_ptr(), view.spans_count) - }); - let read = wrapper_spans - .iter() - .map(|wrapper_span| { - ChatSpanKind::try_from(wrapper_span.kind).map(|kind| ChatSpan { - kind, - bytes: wrapper_span.start..wrapper_span.end, - }) - }) - .collect::, FfiStatusError>>() - .map(|spans_read| ChatSpans::new(view.parsed_length, &spans_read, completeness)) - .map_err(ParseChatMessageError::from); - - unsafe { llama_cpp_bindings_sys::llama_rs_chat_spans_free(spans.as_ptr()) }; - - read -} - -fn chat_session_classify_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_chat_session_classify_status, - out_spans: *mut llama_cpp_bindings_sys::llama_rs_chat_spans, - out_error: *mut c_char, - completeness: ChatOutputCompleteness, -) -> Result { - const OPERATION: &str = "llama_rs_chat_session_classify"; - - match status { - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_OK => NonNull::new(out_spans) - .ok_or_else(|| { - contract_violation(OPERATION, "success status contained null spans").into() - }) - .and_then(|spans| read_chat_spans(spans, completeness)) - .map(ChatClassification::Spans), - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_OUTPUT_DOES_NOT_MATCH_FORMAT => { - Ok(ChatClassification::OutputDoesNotMatchFormat(completeness)) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_THREW_CXX_EXCEPTION => { + contract_violation(OPERATION, "success status contained a null reading").into() + }), + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_READ_LLAMA_CPP_THREW_CXX_EXCEPTION => { Err(ParseChatMessageError::Reported { message: unsafe { read_and_free_cpp_string( @@ -281,25 +161,25 @@ fn chat_session_classify_status_to_result( }?, }) } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_ERROR_STRING_ALLOCATION_FAILED => { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_READ_ERROR_STRING_ALLOCATION_FAILED => { Err(ParseChatMessageError::NotEnoughMemory) } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_OUT_OF_MEMORY => { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_READ_LLAMA_CPP_OUT_OF_MEMORY => { Err(ParseChatMessageError::LlamaCppOutOfMemory) } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_SESSION_ARG => { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_READ_NULL_SESSION_ARG => { Err(contract_violation(OPERATION, "was given a null session argument").into()) } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_TEXT_ARG => { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_READ_NULL_TEXT_ARG => { Err(contract_violation(OPERATION, "was given a null text argument").into()) } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_BYTE_TOKENS_ARG => { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_READ_NULL_BYTE_TOKENS_ARG => { Err(contract_violation(OPERATION, "was given a null byte_tokens argument").into()) } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_SPANS_ARG => { - Err(contract_violation(OPERATION, "was given a null out_spans argument").into()) + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_READ_NULL_OUT_READING_ARG => { + Err(contract_violation(OPERATION, "was given a null out_reading argument").into()) } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_ERROR_ARG => { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_READ_NULL_OUT_ERROR_ARG => { Err(contract_violation(OPERATION, "was given a null out_error argument").into()) } other => Err(FfiStatusError { @@ -310,39 +190,43 @@ fn chat_session_classify_status_to_result( } } -/// One chat request rendered by the model's chat template the way llama-server renders it: the -/// prompt, the grammar that constrains tool calls, and the parser of the model's answer. -pub struct ChatSession<'model> { - model: &'model LlamaModel, +/// One chat request rendered by the model's chat template the way llama-server renders it. +/// +/// The session holds the prompt, the grammar that constrains tool calls, and the parser of the +/// model's answer. It owns everything llama.cpp prepared for the request, so it can be moved to +/// the thread that generates the answer. +pub struct ChatSession { session: NonNull, } -unsafe impl Send for ChatSession<'_> {} +unsafe impl Send for ChatSession {} -impl<'model> ChatSession<'model> { - /// Renders `messages` with `tools_json`, an `OpenAI` tools array, through `templates`: the - /// model's own [`LlamaModel::chat_templates`], or [`ChatTemplates::with_override`]. +impl ChatSession { + /// Renders `messages` and `tools` through `templates`: the model's own + /// [`LlamaModel::chat_templates`], or [`ChatTemplates::with_override`]. /// /// # Errors /// /// Returns the [`ChatSessionError`] describing why the chat template cannot serve the request. pub fn new( - model: &'model LlamaModel, + model: &LlamaModel, templates: &ChatTemplates, - messages: &[LlamaChatMessage], - tools_json: &str, + messages: &[ChatMessage], + tools: &[ChatTool], options: &ChatSessionOptions, ) -> Result { - let messages_json = messages_json(messages, &options.assistant_turn)?; - let tools_json = CString::new(tools_json).map_err(ChatSessionError::RequestContainsNul)?; + let messages_json = messages_json(messages, &options.assistant_turn); + let tools_json = tools_json(tools); let mut out_session: llama_cpp_bindings_sys::llama_rs_chat_session_handle = ptr::null_mut(); let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_chat_session_create( templates.as_ptr(), model.vocab_ptr(), - messages_json.as_ptr(), - tools_json.as_ptr(), + messages_json.as_ptr().cast::(), + messages_json.len(), + tools_json.as_ptr().cast::(), + tools_json.len(), options.into(), &raw mut out_session, &raw mut out_error, @@ -350,7 +234,7 @@ impl<'model> ChatSession<'model> { }; chat_session_create_status_to_result(status, out_session, out_error) - .map(|session| Self { model, session }) + .map(|session| Self { session }) } fn text_of( @@ -425,14 +309,15 @@ impl<'model> ChatSession<'model> { .collect()) } - /// Tokenizes the rendered prompt part by part: control tokens are recognized in the - /// template's own text but never in user input, and BOS is added once before the first part - /// when the vocabulary asks for it. + /// Tokenizes the rendered prompt part by part with the vocabulary of `model`, the model the + /// session was created for: control tokens are recognized in the template's own text but + /// never in user input, and BOS is added once before the first part when the vocabulary asks + /// for it. /// /// # Errors /// /// Returns the [`ChatSessionError`] of reading or tokenizing a prompt part. - pub fn tokenize_prompt(&self) -> Result, ChatSessionError> { + pub fn tokenize_prompt(&self, model: &LlamaModel) -> Result, ChatSessionError> { let mut tokens = Vec::new(); for (part_index, part) in self.prompt_parts()?.iter().enumerate() { @@ -446,7 +331,7 @@ impl<'model> ChatSession<'model> { ChatPromptPart::Input(text) => (text, ParseSpecialTokens::Never), }; - tokens.extend(self.model.str_to_token(text, add_bos, parse_special)?); + tokens.extend(model.str_to_token(text, add_bos, parse_special)?); } Ok(tokens) @@ -583,20 +468,19 @@ impl<'model> ChatSession<'model> { ) } - #[must_use] - pub const fn model(&self) -> &'model LlamaModel { - self.model - } - - /// The sampler that constrains generation to the request's tool calls, or `None` when the - /// request needs no grammar. A grammar that is not lazy starts after the generation prompt, - /// so it accepts the generation-prompt tokens first. + /// The sampler that constrains generation to the request's tool calls with the vocabulary of + /// `model`, the model the session was created for, or `None` when the request needs no + /// grammar. A grammar that is not lazy starts after the generation prompt, so it accepts the + /// generation-prompt tokens first. /// /// # Errors /// /// Returns the [`ChatSessionError`] of reading the grammar, escaping a trigger word, building /// the sampler, or accepting a generation-prompt token. - pub fn grammar_sampler(&self) -> Result, ChatSessionError> { + pub fn grammar_sampler( + &self, + model: &LlamaModel, + ) -> Result, ChatSessionError> { let grammar = self.grammar()?; if grammar.is_empty() { @@ -604,7 +488,7 @@ impl<'model> ChatSession<'model> { } if !self.grammar_is_lazy() { - let mut sampler = LlamaSampler::grammar(self.model, &grammar, GRAMMAR_ROOT)?; + let mut sampler = LlamaSampler::grammar(model, &grammar, GRAMMAR_ROOT)?; for token in self.generation_prompt_tokens() { sampler.accept(token)?; @@ -617,7 +501,7 @@ impl<'model> ChatSession<'model> { LazyGrammarTriggers::of(self.grammar_triggers()?)?; Ok(Some(LlamaSampler::grammar_lazy( - self.model, + model, &grammar, GRAMMAR_ROOT, &patterns, @@ -625,64 +509,38 @@ impl<'model> ChatSession<'model> { )?)) } - /// Parses the model's answer with the template's parser. + /// Reads the model's answer with the template's parser: the reading attributes the answer's + /// bytes to reasoning, content, and tool calls, and builds the parsed message on demand. /// /// # Errors /// /// Returns [`ParseChatMessageError`] when llama.cpp fails while parsing; an answer that does - /// not match the template's format is [`ChatMessageParseOutcome::Unrecognized`]. - pub fn parse( + /// not match the template's format is a reading whose classification says so. + pub fn read( &self, output: &ChatOutput, completeness: ChatOutputCompleteness, - ) -> Result { - let mut out_handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat = ptr::null_mut(); + ) -> Result { + let mut out_reading: llama_cpp_bindings_sys::llama_rs_chat_reading_handle = ptr::null_mut(); let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { - llama_cpp_bindings_sys::llama_rs_chat_session_parse( + llama_cpp_bindings_sys::llama_rs_chat_session_read( self.session.as_ptr(), output.text().as_ptr().cast::(), output.text().len(), output.byte_tokens().as_ptr(), completeness == ChatOutputCompleteness::Partial, - &raw mut out_handle, + &raw mut out_reading, &raw mut out_error, ) }; - chat_session_parse_status_to_result(status, out_handle, out_error, output, completeness) - } - - /// Returns how the template's parser attributes the bytes of the model's answer to - /// reasoning, content, and tool calls. - /// - /// # Errors - /// - /// Returns [`ParseChatMessageError`] when llama.cpp fails while parsing. - pub fn classify( - &self, - output: &ChatOutput, - completeness: ChatOutputCompleteness, - ) -> Result { - let mut out_spans: *mut llama_cpp_bindings_sys::llama_rs_chat_spans = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_chat_session_classify( - self.session.as_ptr(), - output.text().as_ptr().cast::(), - output.text().len(), - output.byte_tokens().as_ptr(), - completeness == ChatOutputCompleteness::Partial, - &raw mut out_spans, - &raw mut out_error, - ) - }; - - chat_session_classify_status_to_result(status, out_spans, out_error, completeness) + chat_session_read_status_to_result(status, out_reading, out_error) + .and_then(|reading| ChatReading::take(reading, output, completeness)) } } -impl Drop for ChatSession<'_> { +impl Drop for ChatSession { fn drop(&mut self) { unsafe { llama_cpp_bindings_sys::llama_rs_chat_session_free(self.session.as_ptr()) }; } @@ -690,38 +548,40 @@ impl Drop for ChatSession<'_> { #[cfg(test)] mod tests { - use llama_cpp_ffi_status::cpp_string_error::CppStringError; - use std::ffi::CStr; - use std::ffi::CString; use std::ptr; - use serde_json::json; - + use llama_cpp_bindings_types::chat_message::ChatMessage; + use llama_cpp_bindings_types::chat_message_content::ChatMessageContent; + use llama_cpp_bindings_types::chat_tool::ChatTool; + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use serde_json::json; - use super::chat_session_classify_status_to_result; use super::chat_session_create_status_to_result; - use super::chat_session_parse_status_to_result; + use super::chat_session_read_status_to_result; use super::messages_json; + use super::tools_json; use crate::chat_assistant_prefill::ChatAssistantPrefill; use crate::chat_assistant_turn::ChatAssistantTurn; - use crate::chat_classification::ChatClassification; - use crate::chat_message_parse_outcome::ChatMessageParseOutcome; - use crate::chat_output::ChatOutput; - use crate::chat_output_completeness::ChatOutputCompleteness; use crate::error::chat_session_error::ChatSessionError; use crate::error::parse_chat_message_error::ParseChatMessageError; - use crate::model::llama_chat_message::LlamaChatMessage; - use crate::raw_chat_message::RawChatMessage; type CreateRefusal = ( llama_cpp_bindings_sys::llama_rs_chat_session_create_status, fn(String) -> ChatSessionError, ); - fn parsed_messages(messages_json: &CStr) -> serde_json::Value { - serde_json::from_str(messages_json.to_str().unwrap()).unwrap() + fn parsed_json(json_text: &str) -> serde_json::Value { + serde_json::from_str(json_text).unwrap() + } + + fn user_says_hi() -> ChatMessage { + ChatMessage::Message { + role: "user".to_owned(), + content: ChatMessageContent::Text("hi".to_owned()), + reasoning_content: None, + } } #[test] @@ -770,15 +630,7 @@ mod tests { #[test] fn creating_a_session_reports_why_llama_cpp_refused_the_request() { - let refusals: [CreateRefusal; 4] = [ - ( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_NOT_JSON, - |message| ChatSessionError::ToolsNotJson { message }, - ), - ( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_INVALID, - |message| ChatSessionError::ToolsInvalid { message }, - ), + let refusals: [CreateRefusal; 2] = [ ( llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TEMPLATE_CANNOT_SERVE_REQUEST, |message| ChatSessionError::TemplateCannotServeRequest { message }, @@ -803,8 +655,6 @@ mod tests { #[test] fn creating_a_session_reports_a_refusal_without_a_message_as_a_contract_violation() { let refusals = [ - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_NOT_JSON, - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_INVALID, llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TEMPLATE_CANNOT_SERVE_REQUEST, llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, ]; @@ -856,232 +706,97 @@ mod tests { } #[test] - fn parsing_an_answer_that_does_not_match_the_format_keeps_the_raw_answer() { - let output = ChatOutput::from("not the format"); - - assert_eq!( - chat_session_parse_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_OUTPUT_DOES_NOT_MATCH_FORMAT, - ptr::null_mut(), - unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"the format differs".as_ptr()) - }, - &output, - ChatOutputCompleteness::Finished, - ), - Ok(ChatMessageParseOutcome::Unrecognized(RawChatMessage { - text: "not the format".to_owned(), - completeness: ChatOutputCompleteness::Finished, - ffi_error_message: "the format differs".to_owned(), - })) - ); - } - - #[test] - fn parsing_an_answer_maps_every_failure_status() { - let output = ChatOutput::default(); - let parse = |status, out_error| { - chat_session_parse_status_to_result( - status, - ptr::null_mut(), - out_error, - &output, - ChatOutputCompleteness::Partial, - ) - .err() + fn reading_an_answer_maps_every_failure_status() { + let read = |status, out_error| { + chat_session_read_status_to_result(status, ptr::null_mut(), out_error).err() }; let contract = |detail| { Some(ParseChatMessageError::FfiContract(FfiContractError { - operation: "llama_rs_chat_session_parse", + operation: "llama_rs_chat_session_read", detail, })) }; assert_eq!( [ - parse( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_OK, + read( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_READ_OK, ptr::null_mut() ), - parse( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION, + read( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_READ_LLAMA_CPP_THREW_CXX_EXCEPTION, unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"the parser threw".as_ptr()) } ), - parse( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_ERROR_STRING_ALLOCATION_FAILED, + read( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_READ_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut() ), - parse( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_OUT_OF_MEMORY, + read( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_READ_ERROR_STRING_ALLOCATION_FAILED, ptr::null_mut() ), - parse( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_SESSION_ARG, + read( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_READ_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut() ), - parse( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_TEXT_ARG, + read( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_READ_NULL_SESSION_ARG, ptr::null_mut() ), - parse( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_BYTE_TOKENS_ARG, + read( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_READ_NULL_TEXT_ARG, ptr::null_mut() ), - parse( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_HANDLE_ARG, - ptr::null_mut() - ), - parse( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_ERROR_ARG, - ptr::null_mut() - ), - parse(255, ptr::null_mut()), - ], - [ - contract("success status contained a null handle"), - Some(ParseChatMessageError::Reported { - message: "the parser threw".to_owned() - }), - Some(ParseChatMessageError::NotEnoughMemory), - Some(ParseChatMessageError::LlamaCppOutOfMemory), - contract("was given a null session argument"), - contract("was given a null text argument"), - contract("was given a null byte_tokens argument"), - contract("was given a null out_handle argument"), - contract("was given a null out_error argument"), - Some(ParseChatMessageError::FfiStatus(FfiStatusError { operation: "llama_rs_chat_session_parse", code: 255 })), - ] - ); - } - - #[test] - fn classifying_an_answer_that_does_not_match_the_format_is_not_an_error() { - assert_eq!( - chat_session_classify_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_OUTPUT_DOES_NOT_MATCH_FORMAT, - ptr::null_mut(), - ptr::null_mut(), - ChatOutputCompleteness::Partial, - ), - Ok(ChatClassification::OutputDoesNotMatchFormat( - ChatOutputCompleteness::Partial - )) - ); - } - - #[test] - fn classifying_an_answer_maps_every_failure_status() { - let classify = |status, out_error| { - chat_session_classify_status_to_result( - status, - ptr::null_mut(), - out_error, - ChatOutputCompleteness::Partial, - ) - .err() - }; - let contract = |detail| { - Some(ParseChatMessageError::FfiContract(FfiContractError { - operation: "llama_rs_chat_session_classify", - detail, - })) - }; - - assert_eq!( - [ - classify( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_OK, + read( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_READ_NULL_BYTE_TOKENS_ARG, ptr::null_mut() ), - classify( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_THREW_CXX_EXCEPTION, - unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"the parser threw".as_ptr()) } - ), - classify( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_ERROR_STRING_ALLOCATION_FAILED, + read( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_READ_NULL_OUT_READING_ARG, ptr::null_mut() ), - classify( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_OUT_OF_MEMORY, + read( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_READ_NULL_OUT_ERROR_ARG, ptr::null_mut() ), - classify( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_SESSION_ARG, - ptr::null_mut() - ), - classify( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_TEXT_ARG, - ptr::null_mut() - ), - classify( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_BYTE_TOKENS_ARG, - ptr::null_mut() - ), - classify( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_SPANS_ARG, - ptr::null_mut() - ), - classify( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_ERROR_ARG, - ptr::null_mut() - ), - classify(255, ptr::null_mut()), + read(255, ptr::null_mut()), ], [ - contract("success status contained null spans"), + contract("success status contained a null reading"), Some(ParseChatMessageError::Reported { message: "the parser threw".to_owned() }), + Some(ParseChatMessageError::CppString(CppStringError::Missing(FfiContractError { + operation: "llama_rs_chat_session_read", + detail: "reported a thrown C++ exception without an error message", + }))), Some(ParseChatMessageError::NotEnoughMemory), Some(ParseChatMessageError::LlamaCppOutOfMemory), contract("was given a null session argument"), contract("was given a null text argument"), contract("was given a null byte_tokens argument"), - contract("was given a null out_spans argument"), + contract("was given a null out_reading argument"), contract("was given a null out_error argument"), - Some(ParseChatMessageError::FfiStatus(FfiStatusError { operation: "llama_rs_chat_session_classify", code: 255 })), + Some(ParseChatMessageError::FfiStatus(FfiStatusError { operation: "llama_rs_chat_session_read", code: 255 })), ] ); } #[test] fn a_new_assistant_turn_adds_no_message() { - let messages = [LlamaChatMessage::new("user".to_owned(), "hi".to_owned()).unwrap()]; - assert_eq!( - parsed_messages(&messages_json(&messages, &ChatAssistantTurn::New).unwrap()), + parsed_json(&messages_json(&[user_says_hi()], &ChatAssistantTurn::New)), json!([{"role": "user", "content": "hi"}]) ); } - #[test] - fn a_message_whose_role_or_content_is_not_utf8_is_refused() { - let not_utf8 = CString::new([0xff]).unwrap(); - let utf8_error = not_utf8.to_str().err(); - let messages = [ - LlamaChatMessage { - role: not_utf8.clone(), - content: c"hi".to_owned(), - }, - LlamaChatMessage { - role: c"user".to_owned(), - content: not_utf8, - }, - ]; - - assert_eq!( - messages.map(|message| messages_json(&[message], &ChatAssistantTurn::New).err()), - [utf8_error, utf8_error].map(|error| error.map(ChatSessionError::MessageNotUtf8)) - ); - } - #[test] fn continuing_reasoning_appends_an_assistant_message_with_that_reasoning() { assert_eq!( - parsed_messages( - &messages_json(&[], &ChatAssistantTurn::ContinueReasoning("hmm".to_owned())) - .unwrap() - ), + parsed_json(&messages_json( + &[], + &ChatAssistantTurn::ContinueReasoning("hmm".to_owned()) + )), json!([{"role": "assistant", "content": "", "reasoning_content": "hmm"}]) ); } @@ -1089,64 +804,43 @@ mod tests { #[test] fn continuing_content_appends_an_assistant_message_with_the_prefill() { assert_eq!( - parsed_messages( - &messages_json( - &[], - &ChatAssistantTurn::ContinueContent(ChatAssistantPrefill { - reasoning_content: "done".to_owned(), - content: "The".to_owned(), - }) - ) - .unwrap() - ), + parsed_json(&messages_json( + &[], + &ChatAssistantTurn::ContinueContent(ChatAssistantPrefill { + reasoning_content: "done".to_owned(), + content: "The".to_owned(), + }) + )), json!([{"role": "assistant", "content": "The", "reasoning_content": "done"}]) ); } #[test] - fn a_parse_or_classification_failure_without_a_message_is_a_contract_violation() { - let output = ChatOutput::default(); + fn a_nul_character_in_a_message_is_escaped_in_the_request() { + let message = ChatMessage::Message { + role: "user".to_owned(), + content: ChatMessageContent::Text("a\0b".to_owned()), + reasoning_content: None, + }; assert_eq!( - [ - chat_session_parse_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_OUTPUT_DOES_NOT_MATCH_FORMAT, - ptr::null_mut(), - ptr::null_mut(), - &output, - ChatOutputCompleteness::Finished, - ) - .err(), - chat_session_parse_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null_mut(), - ptr::null_mut(), - &output, - ChatOutputCompleteness::Finished, - ) - .err(), - chat_session_classify_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null_mut(), - ptr::null_mut(), - ChatOutputCompleteness::Finished, - ) - .err(), - ], - [ - Some(ParseChatMessageError::CppString(CppStringError::Missing(FfiContractError { - operation: "llama_rs_chat_session_parse", - detail: "reported a failure without an error message", - }))), - Some(ParseChatMessageError::CppString(CppStringError::Missing(FfiContractError { - operation: "llama_rs_chat_session_parse", - detail: "reported a failure without an error message", - }))), - Some(ParseChatMessageError::CppString(CppStringError::Missing(FfiContractError { - operation: "llama_rs_chat_session_classify", - detail: "reported a thrown C++ exception without an error message", - }))), - ] + messages_json(&[message], &ChatAssistantTurn::Omitted), + r#"[{"role":"user","content":"a\u0000b"}]"# + ); + } + + #[test] + fn tools_are_an_array_of_openai_functions() { + assert_eq!( + parsed_json(&tools_json(&[ChatTool { + name: "get_time".to_owned(), + description: "Reports the time".to_owned(), + parameters: None, + }])), + json!([{ + "type": "function", + "function": {"name": "get_time", "description": "Reports the time"}, + }]) ); } } diff --git a/llama-cpp-bindings/src/chat_session_attribution.rs b/llama-cpp-bindings/src/chat_session_attribution.rs index da6b294e..1e8c17eb 100644 --- a/llama-cpp-bindings/src/chat_session_attribution.rs +++ b/llama-cpp-bindings/src/chat_session_attribution.rs @@ -1,13 +1,15 @@ use std::collections::VecDeque; use crate::chat_output::ChatOutput; +use crate::chat_reading::ChatReading; use crate::chat_session::ChatSession; use crate::pending_chat_piece::PendingChatPiece; use crate::token::LlamaToken; -pub struct ChatSessionAttribution<'session, 'model> { - pub session: &'session ChatSession<'model>, +pub struct ChatSessionAttribution { + pub session: ChatSession, pub preserved_tokens: Vec, pub output: ChatOutput, pub pending_pieces: VecDeque, + pub latest_reading: ChatReading, } diff --git a/llama-cpp-bindings/src/chat_span_kind.rs b/llama-cpp-bindings/src/chat_span_kind.rs index 1048a530..ba264ca3 100644 --- a/llama-cpp-bindings/src/chat_span_kind.rs +++ b/llama-cpp-bindings/src/chat_span_kind.rs @@ -42,7 +42,7 @@ impl TryFrom for ChatSpanKind { Ok(Self::ToolCallMarker) } other => Err(FfiStatusError { - operation: "llama_rs_chat_spans_view_of", + operation: "llama_rs_chat_reading_spans_view", code: i64::from(other), }), } @@ -112,7 +112,7 @@ mod tests { assert_eq!( ChatSpanKind::try_from(99), Err(FfiStatusError { - operation: "llama_rs_chat_spans_view_of", + operation: "llama_rs_chat_reading_spans_view", code: 99, }) ); diff --git a/llama-cpp-bindings/src/chat_spans.rs b/llama-cpp-bindings/src/chat_spans.rs index 5270e019..129f0250 100644 --- a/llama-cpp-bindings/src/chat_spans.rs +++ b/llama-cpp-bindings/src/chat_spans.rs @@ -100,6 +100,17 @@ impl ChatSpans { }) } + /// The section the parsed output ends in, or [`SampledTokenSection::Pending`] when the parse + /// attributes no text to a section. + #[must_use] + pub fn trailing_section(&self) -> SampledTokenSection { + self.segments + .last() + .map_or(SampledTokenSection::Pending, |segment| { + segment.role.section() + }) + } + /// Returns how llama.cpp's parse attributes the token whose text occupies `bytes` of the /// output, or `None` while the parse has not settled the token yet. #[must_use] @@ -156,6 +167,22 @@ mod tests { ) } + #[test] + fn an_output_ends_in_the_section_of_its_last_span() { + assert_eq!( + reasoning_then_content(ChatOutputCompleteness::Partial).trailing_section(), + SampledTokenSection::Content + ); + } + + #[test] + fn an_output_without_spans_ends_in_no_section() { + assert_eq!( + ChatSpans::new(0, &[], ChatOutputCompleteness::Partial).trailing_section(), + SampledTokenSection::Pending + ); + } + #[test] fn a_token_inside_reasoning_text_is_visible_reasoning() { assert_eq!( diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index 6cb7ce09..a1ab2585 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -281,6 +281,7 @@ pub struct LlamaContext<'model> { limits: ContextLimits, keeps_no_memory: bool, embeddings: bool, + evaluated_tokens: u64, } impl Debug for LlamaContext<'_> { @@ -336,6 +337,7 @@ impl<'model> LlamaContext<'model> { }, keeps_no_memory, embeddings, + evaluated_tokens: 0, }) } @@ -442,12 +444,21 @@ impl<'model> LlamaContext<'model> { } } - /// Records the rows a llama.cpp helper wrote when it decoded on this context without one of - /// this context's batches, such as the mtmd helper decoding media chunks. - pub fn record_helper_decode(&mut self, logits_last: bool) { + /// Records the tokens and the rows a llama.cpp helper evaluated and wrote when it decoded on + /// this context without one of this context's batches, such as the mtmd helper decoding media + /// chunks. + pub fn record_helper_decode(&mut self, evaluated_tokens: u64, logits_last: bool) { + self.evaluated_tokens += evaluated_tokens; self.decoded_outputs.rows = DecodedOutputRows::of_helper_decode(logits_last); } + /// The tokens llama.cpp evaluated on this context: every token of each batch it decoded or + /// encoded successfully, and every token the mtmd helper decoded. + #[must_use] + pub const fn evaluated_tokens(&self) -> u64 { + self.evaluated_tokens + } + fn record_batch_output_rows(&mut self, output_token_indices: &[i32]) { if let DecodedOutputRows::BatchIndices(rows) = &mut self.decoded_outputs.rows { rows.clear(); @@ -492,6 +503,7 @@ impl<'model> LlamaContext<'model> { }; process_status_to_result::(status, out_llama_cpp_return_code, out_error)?; + self.evaluated_tokens += u64::from(batch.n_tokens().unsigned_abs()); self.record_batch_output_rows(batch.output_token_indices()); Ok(()) diff --git a/llama-cpp-bindings/src/error.rs b/llama-cpp-bindings/src/error.rs index 9260a422..91c561a6 100644 --- a/llama-cpp-bindings/src/error.rs +++ b/llama-cpp-bindings/src/error.rs @@ -26,7 +26,6 @@ pub mod memory_seq_pos_max_error; pub mod memory_seq_rm_error; pub mod meta_val_error; pub mod model_params_error; -pub mod new_llama_chat_message_error; pub mod parse_chat_message_error; pub mod process_failure; pub mod sample_error; diff --git a/llama-cpp-bindings/src/error/chat_session_error.rs b/llama-cpp-bindings/src/error/chat_session_error.rs index 593c01b1..83e5ab83 100644 --- a/llama-cpp-bindings/src/error/chat_session_error.rs +++ b/llama-cpp-bindings/src/error/chat_session_error.rs @@ -1,4 +1,3 @@ -use std::ffi::NulError; use std::str::Utf8Error; use llama_cpp_ffi_status::cpp_string_error::CppStringError; @@ -17,14 +16,6 @@ pub enum ChatSessionError { FfiStatus(#[from] FfiStatusError), #[error(transparent)] FfiContract(#[from] FfiContractError), - #[error("a chat message is not valid UTF-8")] - MessageNotUtf8(#[source] Utf8Error), - #[error("the chat request contains an interior NUL byte")] - RequestContainsNul(#[source] NulError), - #[error("the tools are not valid JSON: {message}")] - ToolsNotJson { message: String }, - #[error("the tools are not valid tool definitions: {message}")] - ToolsInvalid { message: String }, #[error("the chat template cannot serve the request: {message}")] TemplateCannotServeRequest { message: String }, #[error("text the chat session holds is not valid UTF-8")] diff --git a/llama-cpp-bindings/src/error/new_llama_chat_message_error.rs b/llama-cpp-bindings/src/error/new_llama_chat_message_error.rs deleted file mode 100644 index 008e7c43..00000000 --- a/llama-cpp-bindings/src/error/new_llama_chat_message_error.rs +++ /dev/null @@ -1,7 +0,0 @@ -use std::ffi::NulError; - -#[derive(Debug, Eq, PartialEq, thiserror::Error)] -pub enum NewLlamaChatMessageError { - #[error("{0}")] - NulError(#[from] NulError), -} diff --git a/llama-cpp-bindings/src/error/parse_chat_message_error.rs b/llama-cpp-bindings/src/error/parse_chat_message_error.rs index 7ecfd485..c2581956 100644 --- a/llama-cpp-bindings/src/error/parse_chat_message_error.rs +++ b/llama-cpp-bindings/src/error/parse_chat_message_error.rs @@ -4,7 +4,7 @@ use llama_cpp_ffi_status::cpp_string_error::CppStringError; use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; -#[derive(Debug, Eq, PartialEq, thiserror::Error)] +#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)] pub enum ParseChatMessageError { #[error(transparent)] CppString(#[from] CppStringError), @@ -18,6 +18,8 @@ pub enum ParseChatMessageError { LlamaCppOutOfMemory, #[error("text llama.cpp parsed out of the answer is not valid UTF-8: {0}")] ParsedTextNotUtf8(#[source] Utf8Error), + #[error("the name of the chat format llama.cpp expected is not valid UTF-8: {0}")] + FormatNameNotUtf8(#[source] Utf8Error), #[error("{message}")] Reported { message: String }, } diff --git a/llama-cpp-bindings/src/lib.rs b/llama-cpp-bindings/src/lib.rs index 17f1f957..1076245a 100644 --- a/llama-cpp-bindings/src/lib.rs +++ b/llama-cpp-bindings/src/lib.rs @@ -14,6 +14,7 @@ pub mod chat_output; pub mod chat_output_completeness; pub mod chat_piece_role; pub mod chat_prompt_part; +pub mod chat_reading; pub mod chat_session; mod chat_session_attribution; pub mod chat_session_options; diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index e2309019..82e815a8 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -1,5 +1,4 @@ pub mod add_bos; -pub mod llama_chat_message; pub mod llama_chat_template; pub mod llama_lazy_mode; pub mod llama_lazy_mode_parse_error; diff --git a/llama-cpp-bindings/src/model/llama_chat_message.rs b/llama-cpp-bindings/src/model/llama_chat_message.rs deleted file mode 100644 index f692f4ec..00000000 --- a/llama-cpp-bindings/src/model/llama_chat_message.rs +++ /dev/null @@ -1,59 +0,0 @@ -use std::ffi::CString; - -use crate::error::new_llama_chat_message_error::NewLlamaChatMessageError; - -#[derive(Debug, Eq, PartialEq, Clone)] -pub struct LlamaChatMessage { - pub role: CString, - pub content: CString, -} - -impl LlamaChatMessage { - /// # Errors - /// If either of ``role`` or ``content`` contain null bytes. - pub fn new(role: String, content: String) -> Result { - Ok(Self { - role: CString::new(role)?, - content: CString::new(content)?, - }) - } -} - -#[cfg(test)] -mod tests { - use std::ffi::CString; - - use crate::error::new_llama_chat_message_error::NewLlamaChatMessageError; - use crate::model::llama_chat_message::LlamaChatMessage; - - #[test] - fn a_role_and_content_without_nul_bytes_form_a_message() { - assert_eq!( - LlamaChatMessage::new("user".to_string(), "hello".to_string()), - Ok(LlamaChatMessage { - role: c"user".to_owned(), - content: c"hello".to_owned(), - }) - ); - } - - #[test] - fn a_nul_byte_in_the_role_is_reported() { - assert_eq!( - LlamaChatMessage::new("us\0er".to_string(), "hello".to_string()), - Err(NewLlamaChatMessageError::NulError( - CString::new("us\0er").unwrap_err() - )) - ); - } - - #[test] - fn a_nul_byte_in_the_content_is_reported() { - assert_eq!( - LlamaChatMessage::new("user".to_string(), "hel\0lo".to_string()), - Err(NewLlamaChatMessageError::NulError( - CString::new("hel\0lo").unwrap_err() - )) - ); - } -} diff --git a/llama-cpp-bindings/src/mtmd/eval_chunk.rs b/llama-cpp-bindings/src/mtmd/eval_chunk.rs index 1b1cb816..40d7145e 100644 --- a/llama-cpp-bindings/src/mtmd/eval_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/eval_chunk.rs @@ -111,7 +111,7 @@ pub fn eval_chunk( out_llama_cpp_return_code, out_error, )?; - llama_ctx.record_helper_decode(logits_last); + llama_ctx.record_helper_decode(chunk.n_tokens() as u64, logits_last); Ok(next_position) } diff --git a/llama-cpp-bindings/src/raw_chat_message.rs b/llama-cpp-bindings/src/raw_chat_message.rs index 1d2bfe4d..19323d54 100644 --- a/llama-cpp-bindings/src/raw_chat_message.rs +++ b/llama-cpp-bindings/src/raw_chat_message.rs @@ -1,8 +1,10 @@ use crate::chat_output_completeness::ChatOutputCompleteness; -#[derive(Debug, Eq, PartialEq)] +/// An answer the template's parser does not recognize, with the name of the format llama.cpp +/// expected it in. +#[derive(Clone, Debug, Eq, PartialEq)] pub struct RawChatMessage { pub text: String, pub completeness: ChatOutputCompleteness, - pub ffi_error_message: String, + pub expected_format: String, } diff --git a/llama-cpp-bindings/src/sampled_token_classifier.rs b/llama-cpp-bindings/src/sampled_token_classifier.rs index 74953811..ac9d1902 100644 --- a/llama-cpp-bindings/src/sampled_token_classifier.rs +++ b/llama-cpp-bindings/src/sampled_token_classifier.rs @@ -9,12 +9,14 @@ use crate::batch_add_error::BatchAddError; use crate::chat_output::ChatOutput; use crate::chat_output_completeness::ChatOutputCompleteness; use crate::chat_piece_role::ChatPieceRole; +use crate::chat_reading::ChatReading; use crate::chat_session::ChatSession; use crate::chat_session_attribution::ChatSessionAttribution; use crate::classified_sample::ClassifiedSample; use crate::context::LlamaContext; use crate::error::eval_multimodal_chunks_error::EvalMultimodalChunksError; use crate::error::ingest_error::IngestError; +use crate::error::parse_chat_message_error::ParseChatMessageError; use crate::error::sample_error::SampleError; use crate::eval_multimodal_chunks_params::EvalMultimodalChunksParams; use crate::generation_progress::GenerationProgress; @@ -48,16 +50,16 @@ const fn outcome_of(token: LlamaToken, role: ChatPieceRole, piece: String) -> In /// Attributes every generated token to reasoning, content, or a tool call, the way llama.cpp's /// parse of the chat template attributes the text the token decoded to, and counts the tokens /// of each section. -pub struct SampledTokenClassifier<'session, 'model> { +pub struct SampledTokenClassifier<'model> { model: &'model LlamaModel, - attribution: TokenAttribution<'session, 'model>, + attribution: TokenAttribution, decoder: encoding_rs::Decoder, section: SampledTokenSection, pending_prompt_tokens: u64, usage: TokenUsage, } -impl<'session, 'model> SampledTokenClassifier<'session, 'model> { +impl<'model> SampledTokenClassifier<'model> { /// A classifier for generation outside of a chat template, whose tokens belong to no /// section. #[must_use] @@ -72,22 +74,34 @@ impl<'session, 'model> SampledTokenClassifier<'session, 'model> { } } - /// A classifier for the answer to `session`. - #[must_use] - pub fn for_chat_session(session: &'session ChatSession<'model>) -> Self { - Self { - model: session.model(), - attribution: TokenAttribution::ChatSession(ChatSessionAttribution { - session, + /// A classifier for the answer to `session`, which was created for `model`. The classifier + /// owns the session for as long as the answer is generated, starting from the session's + /// reading of an answer with nothing generated yet. + /// + /// # Errors + /// + /// Returns [`ParseChatMessageError`] when llama.cpp fails while reading the empty answer. + pub fn for_chat_session( + model: &'model LlamaModel, + session: ChatSession, + ) -> Result { + let output = ChatOutput::default(); + let latest_reading = session.read(&output, ChatOutputCompleteness::Partial)?; + + Ok(Self { + model, + attribution: TokenAttribution::ChatSession(Box::new(ChatSessionAttribution { preserved_tokens: session.preserved_tokens(), - output: ChatOutput::default(), + session, + output, pending_pieces: VecDeque::new(), - }), + latest_reading, + })), decoder: encoding_rs::UTF_8.new_decoder(), section: SampledTokenSection::Pending, pending_prompt_tokens: 0, usage: TokenUsage::new(), - } + }) } /// Classifies a generated token and appends every outcome it settles to `outcomes`. An @@ -156,9 +170,10 @@ impl<'session, 'model> SampledTokenClassifier<'session, 'model> { let TokenAttribution::ChatSession(attribution) = &mut self.attribution else { return Ok(()); }; - let classification = attribution + let reading = attribution .session - .classify(&attribution.output, completeness)?; + .read(&attribution.output, completeness)?; + let classification = reading.classification(); let mut released = Vec::new(); while let Some(pending_piece) = attribution.pending_pieces.pop_front() { @@ -170,6 +185,8 @@ impl<'session, 'model> SampledTokenClassifier<'session, 'model> { released.push(outcome_of(pending_piece.token, role, pending_piece.piece)); } + attribution.latest_reading = reading; + for outcome in released { self.section = outcome.sampled_token.section(); self.record_usage_in(self.section); @@ -357,4 +374,15 @@ impl<'session, 'model> SampledTokenClassifier<'session, 'model> { pub const fn current_section(&self) -> SampledTokenSection { self.section } + + /// The template parser's latest reading of the answer, the one that attributed the most + /// recent outcomes, or `None` for generation outside of a chat template. Parsing it maps + /// that same parse into the answer's message. + #[must_use] + pub fn latest_reading(&self) -> Option<&ChatReading> { + match &self.attribution { + TokenAttribution::Undeterminable => None, + TokenAttribution::ChatSession(attribution) => Some(&attribution.latest_reading), + } + } } diff --git a/llama-cpp-bindings/src/token_attribution.rs b/llama-cpp-bindings/src/token_attribution.rs index c7bffd11..e1f3635f 100644 --- a/llama-cpp-bindings/src/token_attribution.rs +++ b/llama-cpp-bindings/src/token_attribution.rs @@ -1,6 +1,6 @@ use crate::chat_session_attribution::ChatSessionAttribution; -pub enum TokenAttribution<'session, 'model> { +pub enum TokenAttribution { Undeterminable, - ChatSession(ChatSessionAttribution<'session, 'model>), + ChatSession(Box), } diff --git a/llama-cpp-fault-injection/Cargo.toml b/llama-cpp-fault-injection/Cargo.toml index 0e3afbbb..d6423333 100644 --- a/llama-cpp-fault-injection/Cargo.toml +++ b/llama-cpp-fault-injection/Cargo.toml @@ -5,7 +5,6 @@ version.workspace = true edition.workspace = true license.workspace = true repository.workspace = true -publish = false [dependencies] llama-cpp-bindings-sys = { workspace = true, features = ["fault-injection"] }