From 36010caf1c56a522df4c6adf0abd06f1ce9b9d50 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 4 Oct 2026 16:45:00 +0000 Subject: [PATCH] Load a model from memory ced.cpp could only load a model from a file path. A caller that already holds the bytes (a network blob, an archive, or a bundle GGUF with several models) had to write them to a file first. Add two C functions and a C++ method: * ced_capi_load_from_memory(data, size) loads a complete GGUF held in memory. The tensor data is copied during the call, so the caller can free the buffer when it returns. * ced_capi_load_from_memory_prefixed(data, size, prefix) loads a model that is stored inside a larger GGUF, with every key and tensor name prefixed. Only the tensors under the prefix are copied, so a bundle reader does not need a standalone copy of the component. * ced::Ced::load_from_memory(data, size, prefix = ""). The path loader and the memory loaders share everything after the I/O step. Parsing uses ggml's gguf_init_from_buffer, so no temporary file or file descriptor is needed and the code is the same on every platform. The buffer comes from the caller, so it is checked before use. A new precheck walks the header and metadata and rejects input that would make ggml abort the process (for example an empty key name). Metadata values are read only after their type is checked, and every tensor range of a prefixed load is checked against the buffer size. Bad input returns NULL with the reason in ced_capi_last_error(NULL). test_memory compares path, memory and prefixed loads bit for bit, frees the buffer before classifying, rejects truncated and corrupt buffers at many offsets, runs a seeded bit-flip pass over the header, and loads from several threads. It is clean under ASan and UBSan. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- CMakeLists.txt | 1 + README.md | 22 +++ include/ced_capi.h | 30 ++++ src/ced.cpp | 20 ++- src/ced.hpp | 8 + src/ced_capi.cpp | 41 ++++++ src/gguf_check.cpp | 100 +++++++++++++ src/gguf_check.hpp | 20 +++ src/model_loader.cpp | 227 +++++++++++++++++++++------- src/model_loader.hpp | 14 ++ tests/CMakeLists.txt | 7 +- tests/test_memory.cpp | 334 ++++++++++++++++++++++++++++++++++++++++++ 12 files changed, 770 insertions(+), 54 deletions(-) create mode 100644 src/gguf_check.cpp create mode 100644 src/gguf_check.hpp create mode 100644 tests/test_memory.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 52c4bfc..5c82aad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,6 +42,7 @@ endif() set(CED_SRC src/model_loader.cpp + src/gguf_check.cpp src/ced_runner.cpp src/fft.cpp src/mel.cpp diff --git a/README.md b/README.md index 83e185f..2865a00 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,28 @@ ced_capi_free(ctx); The per-PCM entry points take an arbitrary mono window, so a realtime consumer can call them on a sliding buffer for live recognition. There is also a struct-array variant (`ced_capi_classify_pcm`), a WAV-path variant (`ced_capi_classify_path_json`), and `ced_capi_classify_pcm_probs`, which writes every class score in class-index order (no sorting, no allocation) for callers that want the raw distribution. See `include/ced_capi.h` for the full API. +### Loading from memory + +A model can also be loaded from bytes you already hold, with no file path and no temporary file, on Linux, macOS and Windows: + +```c +// `buf` holds a complete GGUF file of `len` bytes (read from a network blob, an archive, a bundle, ...). +ced_ctx *ctx = ced_capi_load_from_memory(buf, len); +free(buf); // fine: the tensor data was copied during the call +if (!ctx) { fprintf(stderr, "%s\n", ced_capi_last_error(NULL)); return 1; } +``` + +The buffer is only read during the call, so you can free or overwrite it as soon as the call returns. For a moment the process holds the buffer and the model together. A truncated or corrupt buffer gives `NULL` and a message in `ced_capi_last_error(NULL)`; it never reads outside `[buf, buf + len)`. Like `ced_capi_load`, loads from several threads at once are safe. + +If the model is one component of a larger GGUF (a bundle that holds several models), pass the whole file and the component prefix: + +```c +// Every key and tensor of the component is stored as "", e.g. "ced.encoder.init_bn.weight". +ced_ctx *ctx = ced_capi_load_from_memory_prefixed(bundle, bundle_len, "ced."); +``` + +Only the tensors under the prefix are copied, so no standalone copy of the component is needed. C++ users can call `ced::Ced::load_from_memory(data, size, prefix = "")`. + --- ## LocalAI diff --git a/include/ced_capi.h b/include/ced_capi.h index 1d3cdb4..5723547 100644 --- a/include/ced_capi.h +++ b/include/ced_capi.h @@ -1,6 +1,8 @@ #ifndef CED_CAPI_H #define CED_CAPI_H +#include + #ifdef __cplusplus extern "C" { #endif @@ -20,12 +22,40 @@ extern "C" { typedef struct ced_ctx ced_ctx; // ABI version. Bump on any breaking change below. v1: initial classify API. +// The *_load_from_memory* functions were added later and do not change the version. int ced_capi_abi_version(void); // Load a GGUF model. Returns an owning context or NULL on failure (on failure, // ced_capi_last_error(NULL) holds the message). Release with ced_capi_free. ced_ctx* ced_capi_load(const char* gguf_path); +// Load a GGUF model from memory instead of a file. `data` points to the bytes of +// a complete GGUF file of `size` bytes. Nothing is read from disk and no +// temporary file is made, on any platform. +// +// Ownership: the buffer is only read during this call. The loader copies the +// tensor data into its own memory, so the caller may free or overwrite `data` +// as soon as the call returns, whatever the result. While the call runs, the +// memory use peaks at about the buffer size plus the model size. +// +// A truncated or corrupt buffer is rejected: the call returns NULL and +// ced_capi_last_error(NULL) holds the reason. It never reads outside +// [data, data + size). Same threading rules as ced_capi_load: the error string +// is thread-local and loads from several threads at once are safe. +// Release the context with ced_capi_free. +ced_ctx* ced_capi_load_from_memory(const void* data, size_t size); + +// As ced_capi_load_from_memory, for a model stored inside a larger GGUF such as +// a bundle. Every metadata key and every tensor name of the model is stored as +// `` (for example prefix "ced." gives "ced.ced.depth" and +// "ced.encoder.init_bn.weight"). Only the tensors under the prefix are copied, +// so the caller can pass the bytes of the whole bundle and no standalone copy of +// the component is needed. `prefix` must not be NULL or empty (use +// ced_capi_load_from_memory for a standalone model). Same ownership rule: the +// buffer may be freed after the call. NULL, with the reason in +// ced_capi_last_error(NULL), if the prefix matches no tensors. +ced_ctx* ced_capi_load_from_memory_prefixed(const void* data, size_t size, const char* prefix); + // Free a context from ced_capi_load. Safe on NULL. void ced_capi_free(ced_ctx* ctx); diff --git a/src/ced.cpp b/src/ced.cpp index a9c9c13..94eaa6a 100644 --- a/src/ced.cpp +++ b/src/ced.cpp @@ -24,9 +24,24 @@ struct Ced::Head { }; bool Ced::load(const std::string& path) { - if (!loader_.load(path)) return false; + err_.clear(); + if (!loader_.load(path)) { err_ = loader_.error(); return false; } + return finish_load(); +} + +bool Ced::load_from_memory(const void* data, size_t size, const std::string& prefix) { + err_.clear(); + if (!loader_.load_from_memory(data, size, prefix)) { err_ = loader_.error(); return false; } + return finish_load(); +} + +// Everything after the I/O step; identical for every load path. +bool Ced::finish_load() { backend_ = std::make_unique(); - if (!backend_->ok() || !loader_.realize_weights(*backend_)) return false; + if (!backend_->ok() || !loader_.realize_weights(*backend_)) { + err_ = "backend or weight setup failed"; + return false; + } const CedConfig& c = loader_.config(); const float* bw = loader_.host_f32("encoder.init_bn.weight"); @@ -35,6 +50,7 @@ bool Ced::load(const std::string& path) { const float* bv = loader_.host_f32("encoder.init_bn.running_var"); if (!bw || !bb || !bm || !bv) { std::fprintf(stderr, "ced: missing init_bn tensors\n"); + err_ = "model is missing the encoder.init_bn tensors"; return false; } bn_scale_.resize(c.n_mels); diff --git a/src/ced.hpp b/src/ced.hpp index f3e50d1..848b93a 100644 --- a/src/ced.hpp +++ b/src/ced.hpp @@ -11,6 +11,12 @@ namespace ced { class Ced { public: bool load(const std::string& path); + // Load from a GGUF in memory (see ModelLoader::load_from_memory). `data` is + // not needed after the call returns. `prefix` selects a model inside a larger + // GGUF; leave it empty for a standalone model. + bool load_from_memory(const void* data, size_t size, const std::string& prefix = ""); + // Why the last load failed ("" if it did not). + const std::string& load_error() const { return err_; } const CedConfig& config() const { return loader_.config(); } // Compute device the model runs on ("cpu", "CUDA0", "Vulkan0", ...). const std::string& device_name() const { return backend_->device_name(); } @@ -46,6 +52,7 @@ class Ced { std::vector& probs, int n_threads = 4); private: + bool finish_load(); struct Embed; struct Head; Embed build_embed(ggml_context* ctx, std::vector& inputs, @@ -53,6 +60,7 @@ class Ced { Head build_blocks(ggml_context* ctx, ggml_tensor* tokens, int n_tokens) const; ModelLoader loader_; + std::string err_; std::unique_ptr backend_; // init_bn (BatchNorm2d, eval) folded into a per-mel scale/shift at load. std::vector bn_scale_, bn_shift_; diff --git a/src/ced_capi.cpp b/src/ced_capi.cpp index 0ce6c8d..3948161 100644 --- a/src/ced_capi.cpp +++ b/src/ced_capi.cpp @@ -5,6 +5,8 @@ #include #include +#include +#include #include #include #include @@ -111,6 +113,45 @@ ced_ctx* ced_capi_load(const char* gguf_path) { return reinterpret_cast(c); } +static ced_ctx* load_memory_impl(const void* data, size_t size, const std::string& prefix) { + if (!data || size == 0) { + g_load_error = "empty model buffer"; + return nullptr; + } + auto* c = new (std::nothrow) CedContext(); + if (!c) { + g_load_error = "out of memory"; + return nullptr; + } + try { + if (!c->model.load_from_memory(data, size, prefix)) { + const std::string& why = c->model.load_error(); + g_load_error = "failed to load model from memory: " + + why; + delete c; + return nullptr; + } + } catch (const std::exception& e) { + g_load_error = std::string("failed to load model from memory: ") + e.what(); + delete c; + return nullptr; + } + g_load_error.clear(); + return reinterpret_cast(c); +} + +ced_ctx* ced_capi_load_from_memory(const void* data, size_t size) { + return load_memory_impl(data, size, std::string()); +} + +ced_ctx* ced_capi_load_from_memory_prefixed(const void* data, size_t size, const char* prefix) { + if (!prefix || !*prefix) { + g_load_error = "null or empty prefix"; + return nullptr; + } + return load_memory_impl(data, size, prefix); +} + void ced_capi_free(ced_ctx* ctx) { delete reinterpret_cast(ctx); } const char* ced_capi_last_error(const ced_ctx* ctx) { diff --git a/src/gguf_check.cpp b/src/gguf_check.cpp new file mode 100644 index 0000000..63fef04 --- /dev/null +++ b/src/gguf_check.cpp @@ -0,0 +1,100 @@ +#include "gguf_check.hpp" + +#include +#include + +namespace ced { + +namespace { + +struct Cursor { + const uint8_t* p; + size_t size; + size_t pos = 0; + + bool have(uint64_t n) const { return n <= size - pos; } + bool skip(uint64_t n) { + if (!have(n)) return false; + pos += (size_t)n; + return true; + } + template + bool read(T* out) { + if (!have(sizeof(T))) return false; + std::memcpy(out, p + pos, sizeof(T)); + pos += sizeof(T); + return true; + } +}; + +// Size in bytes of a fixed-size GGUF value type, 0 for string/array/unknown. +size_t scalar_size(uint32_t t) { + switch (t) { + case 0: case 1: case 7: return 1; // u8, i8, bool + case 2: case 3: return 2; // u16, i16 + case 4: case 5: case 6: return 4; // u32, i32, f32 + case 10: case 11: case 12: return 8; // u64, i64, f64 + default: return 0; + } +} + +constexpr uint32_t kString = 8, kArray = 9; +constexpr uint64_t kMaxString = 1ull << 30; // same cap as ggml's reader + +bool skip_string(Cursor& c) { + uint64_t n; + return c.read(&n) && n <= kMaxString && c.skip(n); +} + +} // namespace + +bool gguf_precheck(const void* data, size_t size, std::string* err) { + auto fail = [&](const char* m) { + if (err) *err = m; + return false; + }; + Cursor c{static_cast(data), size}; + uint32_t version; + int64_t n_tensors, n_kv; + if (!c.have(4) || std::memcmp(c.p, "GGUF", 4) != 0) return fail("not a GGUF file (bad magic)"); + c.pos = 4; + if (!c.read(&version) || (version != 2 && version != 3)) + return fail("unsupported GGUF version"); + if (!c.read(&n_tensors) || !c.read(&n_kv) || n_tensors < 0 || n_kv < 0) + return fail("GGUF header is truncated or has a negative count"); + // Every key/value pair takes at least 13 bytes, so a count that cannot fit is + // corrupt (and must not drive a long loop). + if ((uint64_t)n_kv > (size - c.pos) / 13) return fail("GGUF metadata count exceeds the buffer size"); + + for (int64_t i = 0; i < n_kv; ++i) { + uint64_t klen; + if (!c.read(&klen) || klen > kMaxString || !c.have(klen)) + return fail("GGUF metadata key is truncated"); + if (klen == 0) return fail("GGUF metadata key has an empty name"); + c.skip(klen); + uint32_t type; + if (!c.read(&type)) return fail("GGUF metadata is truncated"); + if (type == kString) { + if (!skip_string(c)) return fail("GGUF string value is truncated"); + } else if (type == kArray) { + uint32_t et; + uint64_t n; + if (!c.read(&et) || !c.read(&n)) return fail("GGUF array header is truncated"); + if (et == kString) { + for (uint64_t j = 0; j < n; ++j) + if (!skip_string(c)) return fail("GGUF string array is truncated"); + } else { + const size_t es = scalar_size(et); + if (es == 0) return fail("GGUF array has an unsupported element type"); + if (n > size / es || !c.skip(n * es)) return fail("GGUF array is truncated"); + } + } else { + const size_t s = scalar_size(type); + if (s == 0) return fail("GGUF metadata has an unknown value type"); + if (!c.skip(s)) return fail("GGUF metadata is truncated"); + } + } + return true; +} + +} // namespace ced diff --git a/src/gguf_check.hpp b/src/gguf_check.hpp new file mode 100644 index 0000000..1bd5e05 --- /dev/null +++ b/src/gguf_check.hpp @@ -0,0 +1,20 @@ +#pragma once +#include +#include + +namespace ced { + +// Cheap structural check of a GGUF held in memory, run BEFORE the buffer is +// handed to ggml's reader. ggml validates sizes and bounds, but it still aborts +// the whole process (GGML_ASSERT) on a few well-formed-looking inputs, such as a +// metadata key with an empty name. A model buffer comes from the caller, so such +// input must become an error instead. +// +// Walks the header and the metadata key/value section with bounds checks on every +// read. Returns false and sets `err` on: a bad magic or version, an empty or +// oversized key, an unknown value type, a nested array, or a value that runs +// past the end of the buffer. It does not look at the tensor table: ggml reports +// those errors without aborting, and the loaders re-check every tensor range. +bool gguf_precheck(const void* data, size_t size, std::string* err); + +} // namespace ced diff --git a/src/model_loader.cpp b/src/model_loader.cpp index 213ddf4..0a462e1 100644 --- a/src/model_loader.cpp +++ b/src/model_loader.cpp @@ -1,33 +1,62 @@ #include "model_loader.hpp" #include "ced_runner.hpp" +#include "gguf_check.hpp" #include "ggml-backend.h" #include "ggml-cpu.h" #include "ggml.h" #include "gguf.h" +#include #include #include +#include +#include namespace ced { -static uint32_t kv_u32(gguf_context* g, const char* k, uint32_t d = 0) { - int64_t id = gguf_find_key(g, k); - return id < 0 ? d : (uint32_t)gguf_get_val_u32(g, id); -} -static float kv_f32(gguf_context* g, const char* k, float d = 0) { - int64_t id = gguf_find_key(g, k); - return id < 0 ? d : gguf_get_val_f32(g, id); -} -static bool kv_bool(gguf_context* g, const char* k, bool d = false) { - int64_t id = gguf_find_key(g, k); - return id < 0 ? d : gguf_get_val_bool(g, id); -} -static std::string kv_str(gguf_context* g, const char* k, const char* d = "") { - int64_t id = gguf_find_key(g, k); - return id < 0 ? std::string(d) : std::string(gguf_get_val_str(g, id)); -} +namespace { + +// Reads typed keys from a gguf_context. Every accessor checks the stored type +// first: ggml aborts on a type or array/scalar mismatch, and a model that comes +// from a caller's memory buffer must not be able to do that. A mismatch is +// remembered in `bad` and reported once by the loader. +struct Kv { + gguf_context* g; + std::string prefix; + std::string bad; + + int64_t find(const char* k, gguf_type want, bool array = false) { + int64_t id = gguf_find_key(g, (prefix + k).c_str()); + if (id < 0) return -1; + const bool is_arr = gguf_get_kv_type(g, id) == GGUF_TYPE_ARRAY; + const gguf_type t = is_arr ? gguf_get_arr_type(g, id) : gguf_get_kv_type(g, id); + if (is_arr != array || t != want) { + if (bad.empty()) bad = "metadata key '" + prefix + k + "' has an unexpected type"; + return -1; + } + return id; + } + uint32_t u32(const char* k, uint32_t d = 0) { + int64_t id = find(k, GGUF_TYPE_UINT32); + return id < 0 ? d : gguf_get_val_u32(g, id); + } + float f32(const char* k, float d = 0) { + int64_t id = find(k, GGUF_TYPE_FLOAT32); + return id < 0 ? d : gguf_get_val_f32(g, id); + } + bool boolean(const char* k, bool d = false) { + int64_t id = find(k, GGUF_TYPE_BOOL); + return id < 0 ? d : gguf_get_val_bool(g, id); + } + std::string str(const char* k, const char* d = "") { + int64_t id = find(k, GGUF_TYPE_STRING); + return id < 0 ? std::string(d) : std::string(gguf_get_val_str(g, id)); + } +}; + +} // namespace ModelLoader::~ModelLoader() { if (weights_buf_) ggml_backend_buffer_free(weights_buf_); @@ -98,54 +127,150 @@ const float* ModelLoader::host_f32(const std::string& n) const { } bool ModelLoader::load(const std::string& path) { + err_.clear(); struct gguf_init_params p { /*no_alloc=*/false, /*ctx=*/&ctx_ }; gguf_ = gguf_init_from_file(path.c_str(), p); if (!gguf_) { + err_ = "cannot open or parse GGUF file: " + path; std::fprintf(stderr, "ced: gguf open failed: %s\n", path.c_str()); return false; } - cfg_.arch = kv_str(gguf_, "ced.arch", "ced"); - cfg_.embed_dim = kv_u32(gguf_, "ced.embed_dim"); - cfg_.depth = kv_u32(gguf_, "ced.depth"); - cfg_.num_heads = kv_u32(gguf_, "ced.num_heads"); - cfg_.outputdim = kv_u32(gguf_, "ced.outputdim"); - cfg_.mlp_ratio = kv_f32(gguf_, "ced.mlp_ratio", 4.0f); - cfg_.qkv_bias = kv_bool(gguf_, "ced.qkv_bias", true); - cfg_.patch_size = kv_u32(gguf_, "ced.patch_size", 16); - cfg_.patch_stride = kv_u32(gguf_, "ced.patch_stride", 16); - cfg_.target_length = kv_u32(gguf_, "ced.target_length"); - cfg_.pooling = kv_str(gguf_, "ced.pooling", "mean"); - cfg_.ln_eps_encoder = kv_f32(gguf_, "ced.ln_eps_encoder", 1e-6f); - cfg_.ln_eps_head = kv_f32(gguf_, "ced.ln_eps_head", 1e-5f); - cfg_.bn_eps = kv_f32(gguf_, "ced.bn_eps", 1e-5f); - cfg_.sample_rate = kv_u32(gguf_, "ced.sample_rate", 16000); - cfg_.n_mels = kv_u32(gguf_, "ced.n_mels"); - cfg_.n_fft = kv_u32(gguf_, "ced.n_fft"); - cfg_.win_size = kv_u32(gguf_, "ced.win_size"); - cfg_.hop_size = kv_u32(gguf_, "ced.hop_size"); - cfg_.n_freqs = kv_u32(gguf_, "ced.n_freqs"); - cfg_.f_min = kv_f32(gguf_, "ced.f_min", 0.0f); - cfg_.f_max = kv_f32(gguf_, "ced.f_max", 8000.0f); - cfg_.center = kv_bool(gguf_, "ced.center", true); - cfg_.a2db_multiplier = kv_f32(gguf_, "ced.a2db_multiplier", 10.0f); - cfg_.a2db_amin = kv_f32(gguf_, "ced.a2db_amin", 1e-10f); - cfg_.a2db_top_db = kv_f32(gguf_, "ced.a2db_top_db", 120.0f); - cfg_.a2db_ref = kv_f32(gguf_, "ced.a2db_ref", 1.0f); + return read_model(""); +} + +bool ModelLoader::load_from_memory(const void* data, size_t size, const std::string& prefix) { + err_.clear(); + if (!data || size == 0) { + err_ = "empty model buffer"; + return false; + } + if (!gguf_precheck(data, size, &err_)) return false; + if (prefix.empty()) { + // ggml copies the tensor data into ctx_; `data` is not used after this call. + struct gguf_init_params p { /*no_alloc=*/false, /*ctx=*/&ctx_ }; + gguf_ = gguf_init_from_buffer(data, size, p); + if (!gguf_) { + err_ = "model buffer is not a valid GGUF file (truncated or corrupt)"; + return false; + } + return read_model(""); + } + + // Prefixed view: parse only the header and tensor table, then copy just the + // tensors under `prefix` into a context of our own, under their stripped names. + ggml_context* meta = nullptr; + struct gguf_init_params p { /*no_alloc=*/true, /*ctx=*/&meta }; + gguf_ = gguf_init_from_buffer(data, size, p); + if (!gguf_) { + err_ = "model buffer is not a valid GGUF file (truncated or corrupt)"; + return false; + } + struct MetaGuard { + ggml_context* c; + ~MetaGuard() { if (c) ggml_free(c); } + } guard{meta}; + + const uint64_t data_off = gguf_get_data_offset(gguf_); + if (data_off > size) { + err_ = "model buffer is truncated (tensor data starts past the end)"; + return false; + } + const uint64_t avail = size - data_off; + const int64_t nt = gguf_get_n_tensors(gguf_); + std::vector ids; + size_t need = 0; + for (int64_t i = 0; i < nt; ++i) { + const char* nm = gguf_get_tensor_name(gguf_, i); + if (std::strncmp(nm, prefix.c_str(), prefix.size()) != 0) continue; + const uint64_t off = gguf_get_tensor_offset(gguf_, i); + const uint64_t nb = gguf_get_tensor_size(gguf_, i); + if (off > avail || nb > avail - off) { + err_ = std::string("model buffer is truncated (tensor ") + nm + " is out of range)"; + return false; + } + ggml_tensor* src = ggml_get_tensor(meta, nm); + if (!src || ggml_nbytes(src) != nb) { + err_ = std::string("tensor table is inconsistent for ") + nm; + return false; + } + need += GGML_PAD((size_t)nb, GGML_MEM_ALIGN) + GGML_MEM_ALIGN + ggml_tensor_overhead(); + ids.push_back(i); + } + if (ids.empty()) { + err_ = "no tensors under prefix '" + prefix + "' in the model buffer"; + return false; + } + struct ggml_init_params ip { need + 4096, nullptr, /*no_alloc=*/false }; + ctx_ = ggml_init(ip); + if (!ctx_) { + err_ = "out of memory"; + return false; + } + const uint8_t* base = static_cast(data) + data_off; + for (int64_t i : ids) { + const char* nm = gguf_get_tensor_name(gguf_, i); + ggml_tensor* src = ggml_get_tensor(meta, nm); + ggml_tensor* t = ggml_new_tensor(ctx_, src->type, GGML_MAX_DIMS, src->ne); + if (!t) { + err_ = "out of memory"; + return false; + } + ggml_set_name(t, nm + prefix.size()); + std::memcpy(t->data, base + gguf_get_tensor_offset(gguf_, i), ggml_nbytes(t)); + } + return read_model(prefix); +} + +bool ModelLoader::read_model(const std::string& prefix) { + Kv kv{gguf_, prefix, {}}; + cfg_.arch = kv.str("ced.arch", "ced"); + cfg_.embed_dim = kv.u32("ced.embed_dim"); + cfg_.depth = kv.u32("ced.depth"); + cfg_.num_heads = kv.u32("ced.num_heads"); + cfg_.outputdim = kv.u32("ced.outputdim"); + cfg_.mlp_ratio = kv.f32("ced.mlp_ratio", 4.0f); + cfg_.qkv_bias = kv.boolean("ced.qkv_bias", true); + cfg_.patch_size = kv.u32("ced.patch_size", 16); + cfg_.patch_stride = kv.u32("ced.patch_stride", 16); + cfg_.target_length = kv.u32("ced.target_length"); + cfg_.pooling = kv.str("ced.pooling", "mean"); + cfg_.ln_eps_encoder = kv.f32("ced.ln_eps_encoder", 1e-6f); + cfg_.ln_eps_head = kv.f32("ced.ln_eps_head", 1e-5f); + cfg_.bn_eps = kv.f32("ced.bn_eps", 1e-5f); + cfg_.sample_rate = kv.u32("ced.sample_rate", 16000); + cfg_.n_mels = kv.u32("ced.n_mels"); + cfg_.n_fft = kv.u32("ced.n_fft"); + cfg_.win_size = kv.u32("ced.win_size"); + cfg_.hop_size = kv.u32("ced.hop_size"); + cfg_.n_freqs = kv.u32("ced.n_freqs"); + cfg_.f_min = kv.f32("ced.f_min", 0.0f); + cfg_.f_max = kv.f32("ced.f_max", 8000.0f); + cfg_.center = kv.boolean("ced.center", true); + cfg_.a2db_multiplier = kv.f32("ced.a2db_multiplier", 10.0f); + cfg_.a2db_amin = kv.f32("ced.a2db_amin", 1e-10f); + cfg_.a2db_top_db = kv.f32("ced.a2db_top_db", 120.0f); + cfg_.a2db_ref = kv.f32("ced.a2db_ref", 1.0f); { - int64_t id = gguf_find_key(gguf_, "ced.labels"); - if (id >= 0 && gguf_get_arr_type(gguf_, id) == GGUF_TYPE_STRING) { + int64_t id = kv.find("ced.labels", GGUF_TYPE_STRING, /*array=*/true); + if (id >= 0) { size_t n = gguf_get_arr_n(gguf_, id); cfg_.labels.resize(n); for (size_t i = 0; i < n; ++i) cfg_.labels[i] = gguf_get_arr_str(gguf_, id, i); } } - const int64_t nt = gguf_get_n_tensors(gguf_); - for (int64_t i = 0; i < nt; ++i) { - const char* nm = gguf_get_tensor_name(gguf_, i); - ggml_tensor* t = ggml_get_tensor(ctx_, nm); - if (t) tensors_[nm] = t; + if (!kv.bad.empty()) { + err_ = kv.bad; + return false; } - return cfg_.embed_dim > 0 && cfg_.depth > 0 && cfg_.outputdim > 0; + // Every tensor of the context (path loads: the whole file; prefixed loads: + // the component, names already stripped). + for (ggml_tensor* t = ggml_get_first_tensor(ctx_); t; t = ggml_get_next_tensor(ctx_, t)) + tensors_[ggml_get_name(t)] = t; + if (!(cfg_.embed_dim > 0 && cfg_.depth > 0 && cfg_.outputdim > 0)) { + err_ = "GGUF is not a ced model (missing ced.embed_dim, ced.depth or ced.outputdim)"; + return false; + } + return true; } ggml_tensor* ModelLoader::tensor(const std::string& n) const { diff --git a/src/model_loader.hpp b/src/model_loader.hpp index 7c406bf..f27c5bb 100644 --- a/src/model_loader.hpp +++ b/src/model_loader.hpp @@ -38,6 +38,17 @@ class ModelLoader { ModelLoader() = default; ~ModelLoader(); bool load(const std::string& path); + // Load from a GGUF held in memory. Nothing is read from disk, and `data` is + // only read during the call: the tensor data is copied into the loader's own + // buffers, so the caller may free or reuse `data` afterwards. + // + // A non-empty `prefix` selects one model inside a larger GGUF (a bundle): + // every metadata key and tensor name of the model is stored as + // ``. Only the tensors under the prefix are copied, and they + // are seen under their unprefixed names. + bool load_from_memory(const void* data, size_t size, const std::string& prefix = ""); + // Why the last load failed ("" if it did not). + const std::string& error() const { return err_; } const CedConfig& config() const { return cfg_; } ggml_tensor* tensor(const std::string& name) const; // nullptr if absent // Make every weight usable as a graph leaf on `backend`. CPU: zero-copy @@ -51,7 +62,10 @@ class ModelLoader { const float* host_f32(const std::string& name) const; private: + bool read_model(const std::string& prefix); // shared by every load path + CedConfig cfg_; + std::string err_; gguf_context* gguf_ = nullptr; ggml_context* ctx_ = nullptr; ggml_context* dev_ctx_ = nullptr; // no_alloc mirror of ctx_ (GPU path) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ba9e679..0eacd6b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,7 +1,7 @@ set(CED_MODEL_GGUF "${PROJECT_SOURCE_DIR}/models/ced-base-f32.gguf") set(CED_BASELINE_GGUF "${PROJECT_SOURCE_DIR}/tests/fixtures/ced-base.baseline.gguf") -foreach(t blocks frontend quant capi) +foreach(t blocks frontend quant capi memory) add_executable(test_${t} test_${t}.cpp) target_link_libraries(test_${t} PRIVATE ced) target_include_directories(test_${t} PRIVATE @@ -9,6 +9,11 @@ foreach(t blocks frontend quant capi) ${PROJECT_SOURCE_DIR}/include ${PROJECT_SOURCE_DIR}/third_party/ggml/include) endforeach() +# Load from memory (plain and prefixed) vs the path loader, bit for bit, plus +# truncated/corrupt buffers. Needs two different CED models (tiny and mini). +add_test(NAME memory COMMAND test_memory + "${CMAKE_SOURCE_DIR}/models/ced-tiny-q8_0.gguf" + "${CMAKE_SOURCE_DIR}/models/ced-mini-q8_0.gguf") add_test(NAME capi COMMAND test_capi "${CED_MODEL_GGUF}" "${CED_BASELINE_GGUF}") # Per-component parity (f32 reference path). diff --git a/tests/test_memory.cpp b/tests/test_memory.cpp new file mode 100644 index 0000000..18e0b4e --- /dev/null +++ b/tests/test_memory.cpp @@ -0,0 +1,334 @@ +// Tests for loading a model from memory (ced_capi_load_from_memory and the +// prefixed variant). No baseline fixture is needed: the reference is the same +// model loaded from its path, and outputs must match bit for bit. +// +// argv[1] = model gguf (any CED size), argv[2] = a second, different model gguf +// (used as the other component of a bundle). +#include "ced_capi.h" + +#include "ggml.h" +#include "gguf.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +int g_fail = 0; +#define CHECK(cond, ...) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s -- ", __FILE__, __LINE__, #cond); \ + std::fprintf(stderr, __VA_ARGS__); \ + std::fprintf(stderr, "\n"); \ + ++g_fail; \ + } \ + } while (0) + +std::vector read_file(const std::string& path) { + std::ifstream f(path, std::ios::binary); + return std::vector((std::istreambuf_iterator(f)), std::istreambuf_iterator()); +} + +// Deterministic 3 s test clip: a few tones plus a noise floor. +std::vector make_wav() { + std::vector w(16000 * 3); + uint32_t s = 12345; + for (size_t i = 0; i < w.size(); ++i) { + s = s * 1664525u + 1013904223u; + float noise = ((s >> 8) / (float)(1 << 24) - 0.5f) * 0.05f; + float t = (float)i / 16000.0f; + w[i] = 0.3f * std::sin(6.2831853f * 440.0f * t) + 0.2f * std::sin(6.2831853f * 1250.0f * t * (1 + t)) + noise; + } + return w; +} + +bool scores(ced_ctx* c, const std::vector& wav, std::vector& out) { + int n = ced_capi_num_classes(c); + out.assign(n, 0.0f); + return ced_capi_classify_pcm_probs(c, wav.data(), (int)wav.size(), 16000, out.data(), n) == n; +} + +bool bitwise_equal(const std::vector& a, const std::vector& b) { + return a.size() == b.size() && std::memcmp(a.data(), b.data(), a.size() * sizeof(float)) == 0; +} + +// Copy `n` bytes of `src` into a buffer of exactly that size, so ASan flags any +// read past the end. +std::vector exact_copy(const std::vector& src, size_t n) { + return std::vector(src.begin(), src.begin() + n); +} + +// ---- in-memory bundle builder ------------------------------------------------ + +void copy_kv(gguf_context* out, gguf_context* g, int64_t i, const std::string& name) { + const char* k = name.c_str(); + switch (gguf_get_kv_type(g, i)) { + case GGUF_TYPE_UINT32: gguf_set_val_u32(out, k, gguf_get_val_u32(g, i)); break; + case GGUF_TYPE_FLOAT32: gguf_set_val_f32(out, k, gguf_get_val_f32(g, i)); break; + case GGUF_TYPE_BOOL: gguf_set_val_bool(out, k, gguf_get_val_bool(g, i)); break; + case GGUF_TYPE_STRING: gguf_set_val_str(out, k, gguf_get_val_str(g, i)); break; + case GGUF_TYPE_ARRAY: { + const gguf_type at = gguf_get_arr_type(g, i); + const size_t n = gguf_get_arr_n(g, i); + if (at == GGUF_TYPE_STRING) { + std::vector v(n); + for (size_t j = 0; j < n; ++j) v[j] = gguf_get_arr_str(g, i, j); + gguf_set_arr_str(out, k, v.data(), n); + } else { + gguf_set_arr_data(out, k, at, gguf_get_arr_data(g, i), n); + } + break; + } + default: break; + } +} + +struct Component { + std::string prefix; + const std::vector* gguf; + std::string retype_key; // if set, this key is written as a float instead +}; + +// Builds one GGUF holding every component, each under its prefix. Keys and +// tensors are renamed ``; tensor bytes are copied unchanged. +std::vector build_bundle(const std::vector& comps) { + gguf_context* out = gguf_init_empty(); + gguf_set_val_str(out, "general.architecture", "test-bundle"); + std::vector srcs; + std::vector ctxs; + ggml_context* tctx = ggml_init({ggml_tensor_overhead() * 4096, nullptr, true}); + std::vector datas; + for (auto& c : comps) { + ggml_context* sc = nullptr; + gguf_context* g = gguf_init_from_buffer(c.gguf->data(), c.gguf->size(), {false, &sc}); + if (!g) { std::fprintf(stderr, "bundle: bad source\n"); std::exit(2); } + srcs.push_back(g); + ctxs.push_back(sc); + for (int64_t i = 0; i < gguf_get_n_kv(g); ++i) { + std::string k = gguf_get_key(g, i); + if (k == "general.alignment") continue; + if (!c.retype_key.empty() && k == c.retype_key) gguf_set_val_f32(out, (c.prefix + k).c_str(), 1.0f); + else copy_kv(out, g, i, c.prefix + k); + } + for (ggml_tensor* t = ggml_get_first_tensor(sc); t; t = ggml_get_next_tensor(sc, t)) { + ggml_tensor* d = ggml_new_tensor(tctx, t->type, GGML_MAX_DIMS, t->ne); + ggml_set_name(d, (c.prefix + ggml_get_name(t)).c_str()); + d->data = t->data; + gguf_add_tensor(out, d); + datas.push_back(t->data); + } + } + const size_t meta = gguf_get_meta_size(out); + std::vector buf(meta, 0); + gguf_get_meta_data(out, buf.data()); + const size_t align = 32; + for (int64_t i = 0; i < gguf_get_n_tensors(out); ++i) { + const size_t off = meta + gguf_get_tensor_offset(out, i); + if (buf.size() < off) buf.resize(off, 0); + const uint8_t* p = (const uint8_t*)datas[i]; + buf.insert(buf.end(), p, p + gguf_get_tensor_size(out, i)); + buf.resize((buf.size() + align - 1) / align * align, 0); + } + for (auto g : srcs) gguf_free(g); + for (auto c : ctxs) ggml_free(c); + ggml_free(tctx); + gguf_free(out); + return buf; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc < 3) { + std::fprintf(stderr, "usage: %s model.gguf other-model.gguf\n", argv[0]); + return 2; + } + const std::string path = argv[1], path2 = argv[2]; + const std::vector file = read_file(path), file2 = read_file(path2); + CHECK(!file.empty() && !file2.empty(), "cannot read models"); + const std::vector wav = make_wav(); + + // Reference: path loader. + ced_ctx* ref = ced_capi_load(path.c_str()); + ced_ctx* ref2 = ced_capi_load(path2.c_str()); + if (!ref || !ref2) { std::fprintf(stderr, "FAIL: path load\n"); return 1; } + std::vector ref_s, ref_s2; + CHECK(scores(ref, wav, ref_s) && scores(ref2, wav, ref_s2), "reference classify"); + CHECK(!bitwise_equal(ref_s, ref_s2), "the two test models must differ"); + + // 1. Same file from memory: identical scores, labels and rate. + { + std::vector heap = file; + ced_ctx* m = ced_capi_load_from_memory(heap.data(), heap.size()); + CHECK(m != nullptr, "load_from_memory: %s", ced_capi_last_error(nullptr)); + if (m) { + // 2. The buffer is not used after the call: wipe and free it first. + std::memset(heap.data(), 0xA5, heap.size()); + std::vector().swap(heap); + std::vector s; + CHECK(scores(m, wav, s) && bitwise_equal(s, ref_s), "memory scores differ from path scores"); + CHECK(ced_capi_num_classes(m) == ced_capi_num_classes(ref), "num_classes"); + CHECK(ced_capi_sample_rate(m) == ced_capi_sample_rate(ref), "sample_rate"); + CHECK(std::strcmp(ced_capi_label(m, 0), ced_capi_label(ref, 0)) == 0, "label 0"); + ced_capi_free(m); + } + } + + // 3. Prefixed view of a two-component bundle, both orders. + for (int order = 0; order < 2; ++order) { + std::vector comps = order == 0 + ? std::vector{{"ced.", &file, ""}, {"other.", &file2, ""}} + : std::vector{{"other.", &file2, ""}, {"ced.", &file, ""}}; + std::vector bundle = build_bundle(comps); + ced_ctx* a = ced_capi_load_from_memory_prefixed(bundle.data(), bundle.size(), "ced."); + ced_ctx* b = ced_capi_load_from_memory_prefixed(bundle.data(), bundle.size(), "other."); + CHECK(a && b, "prefixed load (order %d): %s", order, ced_capi_last_error(nullptr)); + std::memset(bundle.data(), 0x5A, bundle.size()); + std::vector().swap(bundle); + if (a && b) { + std::vector sa, sb; + CHECK(scores(a, wav, sa) && bitwise_equal(sa, ref_s), "prefixed 'ced.' scores (order %d)", order); + CHECK(scores(b, wav, sb) && bitwise_equal(sb, ref_s2), "prefixed 'other.' scores (order %d)", order); + } + ced_capi_free(a); + ced_capi_free(b); + } + + // 4. Bad arguments and the wrong prefix. + CHECK(ced_capi_load_from_memory(nullptr, 100) == nullptr, "NULL data"); + CHECK(std::strlen(ced_capi_last_error(nullptr)) > 0, "error set for NULL data"); + CHECK(ced_capi_load_from_memory(file.data(), 0) == nullptr, "size 0"); + CHECK(ced_capi_load_from_memory_prefixed(file.data(), file.size(), nullptr) == nullptr, "NULL prefix"); + CHECK(ced_capi_load_from_memory_prefixed(file.data(), file.size(), "") == nullptr, "empty prefix"); + { + std::vector bundle = build_bundle({{"ced.", &file, ""}}); + CHECK(ced_capi_load_from_memory_prefixed(bundle.data(), bundle.size(), "nope.") == nullptr, "wrong prefix"); + CHECK(std::strstr(ced_capi_last_error(nullptr), "no tensors under prefix") != nullptr, "wrong prefix message: %s", + ced_capi_last_error(nullptr)); + // A standalone file has no prefix at all. + CHECK(ced_capi_load_from_memory_prefixed(file.data(), file.size(), "ced.") == nullptr, "prefix on standalone file"); + // Not a bundle: loading it without a prefix finds no model keys. + CHECK(ced_capi_load_from_memory(bundle.data(), bundle.size()) == nullptr, "bundle without prefix"); + CHECK(std::strstr(ced_capi_last_error(nullptr), "not a ced model") != nullptr, "bundle w/o prefix message: %s", + ced_capi_last_error(nullptr)); + } + + // 5. A key with the wrong type is an error, not an abort. + { + std::vector bad = build_bundle({{"ced.", &file, "ced.depth"}}); + CHECK(ced_capi_load_from_memory_prefixed(bad.data(), bad.size(), "ced.") == nullptr, "retyped key (prefixed)"); + CHECK(std::strstr(ced_capi_last_error(nullptr), "unexpected type") != nullptr, "message: %s", ced_capi_last_error(nullptr)); + std::vector bad2 = build_bundle({{"", &file, "ced.depth"}}); + CHECK(ced_capi_load_from_memory(bad2.data(), bad2.size()) == nullptr, "retyped key (plain)"); + } + + // 6. Truncation at every region: header (dense), tensor table, tensor data. + { + std::vector bundle = build_bundle({{"ced.", &file, ""}, {"other.", &file2, ""}}); + struct Case { const char* name; const std::vector* buf; const char* prefix; }; + const Case cases[] = {{"plain", &file, nullptr}, {"bundle", &bundle, "ced."}}; + for (const Case& cs : cases) { + const size_t total = cs.buf->size(); + std::vector cuts; + for (size_t n = 0; n < 512 && n < total; ++n) cuts.push_back(n); + for (size_t n = 512; n < 200000 && n < total; n += 257) cuts.push_back(n); + for (int i = 1; i < 120; ++i) cuts.push_back(total * i / 120); + cuts.push_back(total - 1); + int accepted = 0; + for (size_t n : cuts) { + std::vector t = exact_copy(*cs.buf, n); + ced_ctx* c = cs.prefix ? ced_capi_load_from_memory_prefixed(t.data(), t.size(), cs.prefix) + : ced_capi_load_from_memory(t.data(), t.size()); + if (c) { + ++accepted; + ced_capi_free(c); + } else { + CHECK(std::strlen(ced_capi_last_error(nullptr)) > 0, "%s: no error text at %zu", cs.name, n); + } + } + // A plain file cut anywhere must be rejected. A bundle view may succeed + // only once the cut is past the bytes of the selected component. + if (!cs.prefix) CHECK(accepted == 0, "%s: %d truncated buffers accepted", cs.name, accepted); + else std::fprintf(stderr, "[%s] %zu cuts, %d accepted (cut after the component)\n", cs.name, cuts.size(), accepted); + } + // Cut inside the selected component (first in the bundle): must be rejected. + std::vector first = build_bundle({{"ced.", &file, ""}, {"other.", &file2, ""}}); + std::vector t = exact_copy(first, first.size() / 10 + 100000 < first.size() ? first.size() / 10 + 100000 : first.size() / 2); + CHECK(ced_capi_load_from_memory_prefixed(t.data(), t.size(), "ced.") == nullptr, "cut inside component"); + CHECK(std::strstr(ced_capi_last_error(nullptr), "truncated") != nullptr || + std::strstr(ced_capi_last_error(nullptr), "corrupt") != nullptr, "cut message: %s", ced_capi_last_error(nullptr)); + } + + // 7. Corrupt headers. + { + auto expect_reject = [&](const char* what, std::vector b) { + std::vector t = exact_copy(b, b.size()); + CHECK(ced_capi_load_from_memory(t.data(), t.size()) == nullptr, "%s accepted", what); + CHECK(std::strlen(ced_capi_last_error(nullptr)) > 0, "%s: no error text", what); + }; + std::vector b = file; + b[0] ^= 0xff; expect_reject("bad magic", b); + b = file; b[4] = 0xff; b[5] = 0xff; expect_reject("bad version", b); + b = file; std::memset(&b[8], 0xff, 8); expect_reject("huge tensor count", b); + b = file; std::memset(&b[16], 0xff, 8); expect_reject("huge kv count", b); + b = file; std::memset(&b[8], 0, 8); expect_reject("zero tensors", b); + b.assign(file.size(), 0); expect_reject("all zero", b); + b.assign(file.size(), 0xff); expect_reject("all 0xff", b); + b.assign(8, 0); expect_reject("8 zero bytes", b); + } + + // 8. Bit flips in the header and tensor table must never crash. + { + uint32_t s = 777; + auto rnd = [&]() { s = s * 1664525u + 1013904223u; return s >> 8; }; + int ok = 0, rejected = 0; + const size_t span = std::min(file.size(), 4096); + for (int it = 0; it < 400; ++it) { + std::vector b = exact_copy(file, file.size()); + for (int k = 0; k < 1 + (int)(rnd() % 4); ++k) b[rnd() % span] ^= (uint8_t)(1u << (rnd() % 8)); + ced_ctx* c = ced_capi_load_from_memory(b.data(), b.size()); + if (c) { ++ok; ced_capi_free(c); } else ++rejected; + } + std::fprintf(stderr, "[bitflip] %d accepted, %d rejected, no crash\n", ok, rejected); + } + + // 9. Concurrent loads from memory (plain and prefixed) match the reference. + { + std::vector bundle = build_bundle({{"ced.", &file, ""}, {"other.", &file2, ""}}); + std::atomic bad{0}; + std::vector th; + for (int t = 0; t < 6; ++t) + th.emplace_back([&, t]() { + for (int i = 0; i < 3; ++i) { + ced_ctx* c; + const std::vector* want; + if (t % 3 == 0) { c = ced_capi_load_from_memory(file.data(), file.size()); want = &ref_s; } + else if (t % 3 == 1) { c = ced_capi_load_from_memory_prefixed(bundle.data(), bundle.size(), "ced."); want = &ref_s; } + else { c = ced_capi_load_from_memory_prefixed(bundle.data(), bundle.size(), "other."); want = &ref_s2; } + std::vector s; + if (!c || !scores(c, wav, s) || !bitwise_equal(s, *want)) ++bad; + ced_capi_free(c); + // Each thread has its own load error. + if (ced_capi_load_from_memory(nullptr, 0) != nullptr || std::strlen(ced_capi_last_error(nullptr)) == 0) ++bad; + } + }); + for (auto& x : th) x.join(); + CHECK(bad == 0, "%d concurrent failures", bad.load()); + } + + ced_capi_free(ref); + ced_capi_free(ref2); + std::fprintf(stderr, "%s\n", g_fail ? "FAIL" : "PASS"); + return g_fail ? 1 : 0; +}