diff --git a/CMakeLists.txt b/CMakeLists.txt index 52c4bfc..5c82aad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,6 +42,7 @@ endif() set(CED_SRC src/model_loader.cpp + src/gguf_check.cpp src/ced_runner.cpp src/fft.cpp src/mel.cpp diff --git a/README.md b/README.md index 83e185f..2865a00 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,28 @@ ced_capi_free(ctx); The per-PCM entry points take an arbitrary mono window, so a realtime consumer can call them on a sliding buffer for live recognition. There is also a struct-array variant (`ced_capi_classify_pcm`), a WAV-path variant (`ced_capi_classify_path_json`), and `ced_capi_classify_pcm_probs`, which writes every class score in class-index order (no sorting, no allocation) for callers that want the raw distribution. See `include/ced_capi.h` for the full API. +### Loading from memory + +A model can also be loaded from bytes you already hold, with no file path and no temporary file, on Linux, macOS and Windows: + +```c +// `buf` holds a complete GGUF file of `len` bytes (read from a network blob, an archive, a bundle, ...). +ced_ctx *ctx = ced_capi_load_from_memory(buf, len); +free(buf); // fine: the tensor data was copied during the call +if (!ctx) { fprintf(stderr, "%s\n", ced_capi_last_error(NULL)); return 1; } +``` + +The buffer is only read during the call, so you can free or overwrite it as soon as the call returns. For a moment the process holds the buffer and the model together. A truncated or corrupt buffer gives `NULL` and a message in `ced_capi_last_error(NULL)`; it never reads outside `[buf, buf + len)`. Like `ced_capi_load`, loads from several threads at once are safe. + +If the model is one component of a larger GGUF (a bundle that holds several models), pass the whole file and the component prefix: + +```c +// Every key and tensor of the component is stored as "", e.g. "ced.encoder.init_bn.weight". +ced_ctx *ctx = ced_capi_load_from_memory_prefixed(bundle, bundle_len, "ced."); +``` + +Only the tensors under the prefix are copied, so no standalone copy of the component is needed. C++ users can call `ced::Ced::load_from_memory(data, size, prefix = "")`. + --- ## LocalAI diff --git a/include/ced_capi.h b/include/ced_capi.h index 1d3cdb4..5723547 100644 --- a/include/ced_capi.h +++ b/include/ced_capi.h @@ -1,6 +1,8 @@ #ifndef CED_CAPI_H #define CED_CAPI_H +#include + #ifdef __cplusplus extern "C" { #endif @@ -20,12 +22,40 @@ extern "C" { typedef struct ced_ctx ced_ctx; // ABI version. Bump on any breaking change below. v1: initial classify API. +// The *_load_from_memory* functions were added later and do not change the version. int ced_capi_abi_version(void); // Load a GGUF model. Returns an owning context or NULL on failure (on failure, // ced_capi_last_error(NULL) holds the message). Release with ced_capi_free. ced_ctx* ced_capi_load(const char* gguf_path); +// Load a GGUF model from memory instead of a file. `data` points to the bytes of +// a complete GGUF file of `size` bytes. Nothing is read from disk and no +// temporary file is made, on any platform. +// +// Ownership: the buffer is only read during this call. The loader copies the +// tensor data into its own memory, so the caller may free or overwrite `data` +// as soon as the call returns, whatever the result. While the call runs, the +// memory use peaks at about the buffer size plus the model size. +// +// A truncated or corrupt buffer is rejected: the call returns NULL and +// ced_capi_last_error(NULL) holds the reason. It never reads outside +// [data, data + size). Same threading rules as ced_capi_load: the error string +// is thread-local and loads from several threads at once are safe. +// Release the context with ced_capi_free. +ced_ctx* ced_capi_load_from_memory(const void* data, size_t size); + +// As ced_capi_load_from_memory, for a model stored inside a larger GGUF such as +// a bundle. Every metadata key and every tensor name of the model is stored as +// `` (for example prefix "ced." gives "ced.ced.depth" and +// "ced.encoder.init_bn.weight"). Only the tensors under the prefix are copied, +// so the caller can pass the bytes of the whole bundle and no standalone copy of +// the component is needed. `prefix` must not be NULL or empty (use +// ced_capi_load_from_memory for a standalone model). Same ownership rule: the +// buffer may be freed after the call. NULL, with the reason in +// ced_capi_last_error(NULL), if the prefix matches no tensors. +ced_ctx* ced_capi_load_from_memory_prefixed(const void* data, size_t size, const char* prefix); + // Free a context from ced_capi_load. Safe on NULL. void ced_capi_free(ced_ctx* ctx); diff --git a/src/ced.cpp b/src/ced.cpp index a9c9c13..94eaa6a 100644 --- a/src/ced.cpp +++ b/src/ced.cpp @@ -24,9 +24,24 @@ struct Ced::Head { }; bool Ced::load(const std::string& path) { - if (!loader_.load(path)) return false; + err_.clear(); + if (!loader_.load(path)) { err_ = loader_.error(); return false; } + return finish_load(); +} + +bool Ced::load_from_memory(const void* data, size_t size, const std::string& prefix) { + err_.clear(); + if (!loader_.load_from_memory(data, size, prefix)) { err_ = loader_.error(); return false; } + return finish_load(); +} + +// Everything after the I/O step; identical for every load path. +bool Ced::finish_load() { backend_ = std::make_unique(); - if (!backend_->ok() || !loader_.realize_weights(*backend_)) return false; + if (!backend_->ok() || !loader_.realize_weights(*backend_)) { + err_ = "backend or weight setup failed"; + return false; + } const CedConfig& c = loader_.config(); const float* bw = loader_.host_f32("encoder.init_bn.weight"); @@ -35,6 +50,7 @@ bool Ced::load(const std::string& path) { const float* bv = loader_.host_f32("encoder.init_bn.running_var"); if (!bw || !bb || !bm || !bv) { std::fprintf(stderr, "ced: missing init_bn tensors\n"); + err_ = "model is missing the encoder.init_bn tensors"; return false; } bn_scale_.resize(c.n_mels); diff --git a/src/ced.hpp b/src/ced.hpp index f3e50d1..848b93a 100644 --- a/src/ced.hpp +++ b/src/ced.hpp @@ -11,6 +11,12 @@ namespace ced { class Ced { public: bool load(const std::string& path); + // Load from a GGUF in memory (see ModelLoader::load_from_memory). `data` is + // not needed after the call returns. `prefix` selects a model inside a larger + // GGUF; leave it empty for a standalone model. + bool load_from_memory(const void* data, size_t size, const std::string& prefix = ""); + // Why the last load failed ("" if it did not). + const std::string& load_error() const { return err_; } const CedConfig& config() const { return loader_.config(); } // Compute device the model runs on ("cpu", "CUDA0", "Vulkan0", ...). const std::string& device_name() const { return backend_->device_name(); } @@ -46,6 +52,7 @@ class Ced { std::vector& probs, int n_threads = 4); private: + bool finish_load(); struct Embed; struct Head; Embed build_embed(ggml_context* ctx, std::vector& inputs, @@ -53,6 +60,7 @@ class Ced { Head build_blocks(ggml_context* ctx, ggml_tensor* tokens, int n_tokens) const; ModelLoader loader_; + std::string err_; std::unique_ptr backend_; // init_bn (BatchNorm2d, eval) folded into a per-mel scale/shift at load. std::vector bn_scale_, bn_shift_; diff --git a/src/ced_capi.cpp b/src/ced_capi.cpp index 0ce6c8d..3948161 100644 --- a/src/ced_capi.cpp +++ b/src/ced_capi.cpp @@ -5,6 +5,8 @@ #include #include +#include +#include #include #include #include @@ -111,6 +113,45 @@ ced_ctx* ced_capi_load(const char* gguf_path) { return reinterpret_cast(c); } +static ced_ctx* load_memory_impl(const void* data, size_t size, const std::string& prefix) { + if (!data || size == 0) { + g_load_error = "empty model buffer"; + return nullptr; + } + auto* c = new (std::nothrow) CedContext(); + if (!c) { + g_load_error = "out of memory"; + return nullptr; + } + try { + if (!c->model.load_from_memory(data, size, prefix)) { + const std::string& why = c->model.load_error(); + g_load_error = "failed to load model from memory: " + + why; + delete c; + return nullptr; + } + } catch (const std::exception& e) { + g_load_error = std::string("failed to load model from memory: ") + e.what(); + delete c; + return nullptr; + } + g_load_error.clear(); + return reinterpret_cast(c); +} + +ced_ctx* ced_capi_load_from_memory(const void* data, size_t size) { + return load_memory_impl(data, size, std::string()); +} + +ced_ctx* ced_capi_load_from_memory_prefixed(const void* data, size_t size, const char* prefix) { + if (!prefix || !*prefix) { + g_load_error = "null or empty prefix"; + return nullptr; + } + return load_memory_impl(data, size, prefix); +} + void ced_capi_free(ced_ctx* ctx) { delete reinterpret_cast(ctx); } const char* ced_capi_last_error(const ced_ctx* ctx) { diff --git a/src/gguf_check.cpp b/src/gguf_check.cpp new file mode 100644 index 0000000..63fef04 --- /dev/null +++ b/src/gguf_check.cpp @@ -0,0 +1,100 @@ +#include "gguf_check.hpp" + +#include +#include + +namespace ced { + +namespace { + +struct Cursor { + const uint8_t* p; + size_t size; + size_t pos = 0; + + bool have(uint64_t n) const { return n <= size - pos; } + bool skip(uint64_t n) { + if (!have(n)) return false; + pos += (size_t)n; + return true; + } + template + bool read(T* out) { + if (!have(sizeof(T))) return false; + std::memcpy(out, p + pos, sizeof(T)); + pos += sizeof(T); + return true; + } +}; + +// Size in bytes of a fixed-size GGUF value type, 0 for string/array/unknown. +size_t scalar_size(uint32_t t) { + switch (t) { + case 0: case 1: case 7: return 1; // u8, i8, bool + case 2: case 3: return 2; // u16, i16 + case 4: case 5: case 6: return 4; // u32, i32, f32 + case 10: case 11: case 12: return 8; // u64, i64, f64 + default: return 0; + } +} + +constexpr uint32_t kString = 8, kArray = 9; +constexpr uint64_t kMaxString = 1ull << 30; // same cap as ggml's reader + +bool skip_string(Cursor& c) { + uint64_t n; + return c.read(&n) && n <= kMaxString && c.skip(n); +} + +} // namespace + +bool gguf_precheck(const void* data, size_t size, std::string* err) { + auto fail = [&](const char* m) { + if (err) *err = m; + return false; + }; + Cursor c{static_cast(data), size}; + uint32_t version; + int64_t n_tensors, n_kv; + if (!c.have(4) || std::memcmp(c.p, "GGUF", 4) != 0) return fail("not a GGUF file (bad magic)"); + c.pos = 4; + if (!c.read(&version) || (version != 2 && version != 3)) + return fail("unsupported GGUF version"); + if (!c.read(&n_tensors) || !c.read(&n_kv) || n_tensors < 0 || n_kv < 0) + return fail("GGUF header is truncated or has a negative count"); + // Every key/value pair takes at least 13 bytes, so a count that cannot fit is + // corrupt (and must not drive a long loop). + if ((uint64_t)n_kv > (size - c.pos) / 13) return fail("GGUF metadata count exceeds the buffer size"); + + for (int64_t i = 0; i < n_kv; ++i) { + uint64_t klen; + if (!c.read(&klen) || klen > kMaxString || !c.have(klen)) + return fail("GGUF metadata key is truncated"); + if (klen == 0) return fail("GGUF metadata key has an empty name"); + c.skip(klen); + uint32_t type; + if (!c.read(&type)) return fail("GGUF metadata is truncated"); + if (type == kString) { + if (!skip_string(c)) return fail("GGUF string value is truncated"); + } else if (type == kArray) { + uint32_t et; + uint64_t n; + if (!c.read(&et) || !c.read(&n)) return fail("GGUF array header is truncated"); + if (et == kString) { + for (uint64_t j = 0; j < n; ++j) + if (!skip_string(c)) return fail("GGUF string array is truncated"); + } else { + const size_t es = scalar_size(et); + if (es == 0) return fail("GGUF array has an unsupported element type"); + if (n > size / es || !c.skip(n * es)) return fail("GGUF array is truncated"); + } + } else { + const size_t s = scalar_size(type); + if (s == 0) return fail("GGUF metadata has an unknown value type"); + if (!c.skip(s)) return fail("GGUF metadata is truncated"); + } + } + return true; +} + +} // namespace ced diff --git a/src/gguf_check.hpp b/src/gguf_check.hpp new file mode 100644 index 0000000..1bd5e05 --- /dev/null +++ b/src/gguf_check.hpp @@ -0,0 +1,20 @@ +#pragma once +#include +#include + +namespace ced { + +// Cheap structural check of a GGUF held in memory, run BEFORE the buffer is +// handed to ggml's reader. ggml validates sizes and bounds, but it still aborts +// the whole process (GGML_ASSERT) on a few well-formed-looking inputs, such as a +// metadata key with an empty name. A model buffer comes from the caller, so such +// input must become an error instead. +// +// Walks the header and the metadata key/value section with bounds checks on every +// read. Returns false and sets `err` on: a bad magic or version, an empty or +// oversized key, an unknown value type, a nested array, or a value that runs +// past the end of the buffer. It does not look at the tensor table: ggml reports +// those errors without aborting, and the loaders re-check every tensor range. +bool gguf_precheck(const void* data, size_t size, std::string* err); + +} // namespace ced diff --git a/src/model_loader.cpp b/src/model_loader.cpp index 213ddf4..0a462e1 100644 --- a/src/model_loader.cpp +++ b/src/model_loader.cpp @@ -1,33 +1,62 @@ #include "model_loader.hpp" #include "ced_runner.hpp" +#include "gguf_check.hpp" #include "ggml-backend.h" #include "ggml-cpu.h" #include "ggml.h" #include "gguf.h" +#include #include #include +#include +#include namespace ced { -static uint32_t kv_u32(gguf_context* g, const char* k, uint32_t d = 0) { - int64_t id = gguf_find_key(g, k); - return id < 0 ? d : (uint32_t)gguf_get_val_u32(g, id); -} -static float kv_f32(gguf_context* g, const char* k, float d = 0) { - int64_t id = gguf_find_key(g, k); - return id < 0 ? d : gguf_get_val_f32(g, id); -} -static bool kv_bool(gguf_context* g, const char* k, bool d = false) { - int64_t id = gguf_find_key(g, k); - return id < 0 ? d : gguf_get_val_bool(g, id); -} -static std::string kv_str(gguf_context* g, const char* k, const char* d = "") { - int64_t id = gguf_find_key(g, k); - return id < 0 ? std::string(d) : std::string(gguf_get_val_str(g, id)); -} +namespace { + +// Reads typed keys from a gguf_context. Every accessor checks the stored type +// first: ggml aborts on a type or array/scalar mismatch, and a model that comes +// from a caller's memory buffer must not be able to do that. A mismatch is +// remembered in `bad` and reported once by the loader. +struct Kv { + gguf_context* g; + std::string prefix; + std::string bad; + + int64_t find(const char* k, gguf_type want, bool array = false) { + int64_t id = gguf_find_key(g, (prefix + k).c_str()); + if (id < 0) return -1; + const bool is_arr = gguf_get_kv_type(g, id) == GGUF_TYPE_ARRAY; + const gguf_type t = is_arr ? gguf_get_arr_type(g, id) : gguf_get_kv_type(g, id); + if (is_arr != array || t != want) { + if (bad.empty()) bad = "metadata key '" + prefix + k + "' has an unexpected type"; + return -1; + } + return id; + } + uint32_t u32(const char* k, uint32_t d = 0) { + int64_t id = find(k, GGUF_TYPE_UINT32); + return id < 0 ? d : gguf_get_val_u32(g, id); + } + float f32(const char* k, float d = 0) { + int64_t id = find(k, GGUF_TYPE_FLOAT32); + return id < 0 ? d : gguf_get_val_f32(g, id); + } + bool boolean(const char* k, bool d = false) { + int64_t id = find(k, GGUF_TYPE_BOOL); + return id < 0 ? d : gguf_get_val_bool(g, id); + } + std::string str(const char* k, const char* d = "") { + int64_t id = find(k, GGUF_TYPE_STRING); + return id < 0 ? std::string(d) : std::string(gguf_get_val_str(g, id)); + } +}; + +} // namespace ModelLoader::~ModelLoader() { if (weights_buf_) ggml_backend_buffer_free(weights_buf_); @@ -98,54 +127,150 @@ const float* ModelLoader::host_f32(const std::string& n) const { } bool ModelLoader::load(const std::string& path) { + err_.clear(); struct gguf_init_params p { /*no_alloc=*/false, /*ctx=*/&ctx_ }; gguf_ = gguf_init_from_file(path.c_str(), p); if (!gguf_) { + err_ = "cannot open or parse GGUF file: " + path; std::fprintf(stderr, "ced: gguf open failed: %s\n", path.c_str()); return false; } - cfg_.arch = kv_str(gguf_, "ced.arch", "ced"); - cfg_.embed_dim = kv_u32(gguf_, "ced.embed_dim"); - cfg_.depth = kv_u32(gguf_, "ced.depth"); - cfg_.num_heads = kv_u32(gguf_, "ced.num_heads"); - cfg_.outputdim = kv_u32(gguf_, "ced.outputdim"); - cfg_.mlp_ratio = kv_f32(gguf_, "ced.mlp_ratio", 4.0f); - cfg_.qkv_bias = kv_bool(gguf_, "ced.qkv_bias", true); - cfg_.patch_size = kv_u32(gguf_, "ced.patch_size", 16); - cfg_.patch_stride = kv_u32(gguf_, "ced.patch_stride", 16); - cfg_.target_length = kv_u32(gguf_, "ced.target_length"); - cfg_.pooling = kv_str(gguf_, "ced.pooling", "mean"); - cfg_.ln_eps_encoder = kv_f32(gguf_, "ced.ln_eps_encoder", 1e-6f); - cfg_.ln_eps_head = kv_f32(gguf_, "ced.ln_eps_head", 1e-5f); - cfg_.bn_eps = kv_f32(gguf_, "ced.bn_eps", 1e-5f); - cfg_.sample_rate = kv_u32(gguf_, "ced.sample_rate", 16000); - cfg_.n_mels = kv_u32(gguf_, "ced.n_mels"); - cfg_.n_fft = kv_u32(gguf_, "ced.n_fft"); - cfg_.win_size = kv_u32(gguf_, "ced.win_size"); - cfg_.hop_size = kv_u32(gguf_, "ced.hop_size"); - cfg_.n_freqs = kv_u32(gguf_, "ced.n_freqs"); - cfg_.f_min = kv_f32(gguf_, "ced.f_min", 0.0f); - cfg_.f_max = kv_f32(gguf_, "ced.f_max", 8000.0f); - cfg_.center = kv_bool(gguf_, "ced.center", true); - cfg_.a2db_multiplier = kv_f32(gguf_, "ced.a2db_multiplier", 10.0f); - cfg_.a2db_amin = kv_f32(gguf_, "ced.a2db_amin", 1e-10f); - cfg_.a2db_top_db = kv_f32(gguf_, "ced.a2db_top_db", 120.0f); - cfg_.a2db_ref = kv_f32(gguf_, "ced.a2db_ref", 1.0f); + return read_model(""); +} + +bool ModelLoader::load_from_memory(const void* data, size_t size, const std::string& prefix) { + err_.clear(); + if (!data || size == 0) { + err_ = "empty model buffer"; + return false; + } + if (!gguf_precheck(data, size, &err_)) return false; + if (prefix.empty()) { + // ggml copies the tensor data into ctx_; `data` is not used after this call. + struct gguf_init_params p { /*no_alloc=*/false, /*ctx=*/&ctx_ }; + gguf_ = gguf_init_from_buffer(data, size, p); + if (!gguf_) { + err_ = "model buffer is not a valid GGUF file (truncated or corrupt)"; + return false; + } + return read_model(""); + } + + // Prefixed view: parse only the header and tensor table, then copy just the + // tensors under `prefix` into a context of our own, under their stripped names. + ggml_context* meta = nullptr; + struct gguf_init_params p { /*no_alloc=*/true, /*ctx=*/&meta }; + gguf_ = gguf_init_from_buffer(data, size, p); + if (!gguf_) { + err_ = "model buffer is not a valid GGUF file (truncated or corrupt)"; + return false; + } + struct MetaGuard { + ggml_context* c; + ~MetaGuard() { if (c) ggml_free(c); } + } guard{meta}; + + const uint64_t data_off = gguf_get_data_offset(gguf_); + if (data_off > size) { + err_ = "model buffer is truncated (tensor data starts past the end)"; + return false; + } + const uint64_t avail = size - data_off; + const int64_t nt = gguf_get_n_tensors(gguf_); + std::vector ids; + size_t need = 0; + for (int64_t i = 0; i < nt; ++i) { + const char* nm = gguf_get_tensor_name(gguf_, i); + if (std::strncmp(nm, prefix.c_str(), prefix.size()) != 0) continue; + const uint64_t off = gguf_get_tensor_offset(gguf_, i); + const uint64_t nb = gguf_get_tensor_size(gguf_, i); + if (off > avail || nb > avail - off) { + err_ = std::string("model buffer is truncated (tensor ") + nm + " is out of range)"; + return false; + } + ggml_tensor* src = ggml_get_tensor(meta, nm); + if (!src || ggml_nbytes(src) != nb) { + err_ = std::string("tensor table is inconsistent for ") + nm; + return false; + } + need += GGML_PAD((size_t)nb, GGML_MEM_ALIGN) + GGML_MEM_ALIGN + ggml_tensor_overhead(); + ids.push_back(i); + } + if (ids.empty()) { + err_ = "no tensors under prefix '" + prefix + "' in the model buffer"; + return false; + } + struct ggml_init_params ip { need + 4096, nullptr, /*no_alloc=*/false }; + ctx_ = ggml_init(ip); + if (!ctx_) { + err_ = "out of memory"; + return false; + } + const uint8_t* base = static_cast(data) + data_off; + for (int64_t i : ids) { + const char* nm = gguf_get_tensor_name(gguf_, i); + ggml_tensor* src = ggml_get_tensor(meta, nm); + ggml_tensor* t = ggml_new_tensor(ctx_, src->type, GGML_MAX_DIMS, src->ne); + if (!t) { + err_ = "out of memory"; + return false; + } + ggml_set_name(t, nm + prefix.size()); + std::memcpy(t->data, base + gguf_get_tensor_offset(gguf_, i), ggml_nbytes(t)); + } + return read_model(prefix); +} + +bool ModelLoader::read_model(const std::string& prefix) { + Kv kv{gguf_, prefix, {}}; + cfg_.arch = kv.str("ced.arch", "ced"); + cfg_.embed_dim = kv.u32("ced.embed_dim"); + cfg_.depth = kv.u32("ced.depth"); + cfg_.num_heads = kv.u32("ced.num_heads"); + cfg_.outputdim = kv.u32("ced.outputdim"); + cfg_.mlp_ratio = kv.f32("ced.mlp_ratio", 4.0f); + cfg_.qkv_bias = kv.boolean("ced.qkv_bias", true); + cfg_.patch_size = kv.u32("ced.patch_size", 16); + cfg_.patch_stride = kv.u32("ced.patch_stride", 16); + cfg_.target_length = kv.u32("ced.target_length"); + cfg_.pooling = kv.str("ced.pooling", "mean"); + cfg_.ln_eps_encoder = kv.f32("ced.ln_eps_encoder", 1e-6f); + cfg_.ln_eps_head = kv.f32("ced.ln_eps_head", 1e-5f); + cfg_.bn_eps = kv.f32("ced.bn_eps", 1e-5f); + cfg_.sample_rate = kv.u32("ced.sample_rate", 16000); + cfg_.n_mels = kv.u32("ced.n_mels"); + cfg_.n_fft = kv.u32("ced.n_fft"); + cfg_.win_size = kv.u32("ced.win_size"); + cfg_.hop_size = kv.u32("ced.hop_size"); + cfg_.n_freqs = kv.u32("ced.n_freqs"); + cfg_.f_min = kv.f32("ced.f_min", 0.0f); + cfg_.f_max = kv.f32("ced.f_max", 8000.0f); + cfg_.center = kv.boolean("ced.center", true); + cfg_.a2db_multiplier = kv.f32("ced.a2db_multiplier", 10.0f); + cfg_.a2db_amin = kv.f32("ced.a2db_amin", 1e-10f); + cfg_.a2db_top_db = kv.f32("ced.a2db_top_db", 120.0f); + cfg_.a2db_ref = kv.f32("ced.a2db_ref", 1.0f); { - int64_t id = gguf_find_key(gguf_, "ced.labels"); - if (id >= 0 && gguf_get_arr_type(gguf_, id) == GGUF_TYPE_STRING) { + int64_t id = kv.find("ced.labels", GGUF_TYPE_STRING, /*array=*/true); + if (id >= 0) { size_t n = gguf_get_arr_n(gguf_, id); cfg_.labels.resize(n); for (size_t i = 0; i < n; ++i) cfg_.labels[i] = gguf_get_arr_str(gguf_, id, i); } } - const int64_t nt = gguf_get_n_tensors(gguf_); - for (int64_t i = 0; i < nt; ++i) { - const char* nm = gguf_get_tensor_name(gguf_, i); - ggml_tensor* t = ggml_get_tensor(ctx_, nm); - if (t) tensors_[nm] = t; + if (!kv.bad.empty()) { + err_ = kv.bad; + return false; } - return cfg_.embed_dim > 0 && cfg_.depth > 0 && cfg_.outputdim > 0; + // Every tensor of the context (path loads: the whole file; prefixed loads: + // the component, names already stripped). + for (ggml_tensor* t = ggml_get_first_tensor(ctx_); t; t = ggml_get_next_tensor(ctx_, t)) + tensors_[ggml_get_name(t)] = t; + if (!(cfg_.embed_dim > 0 && cfg_.depth > 0 && cfg_.outputdim > 0)) { + err_ = "GGUF is not a ced model (missing ced.embed_dim, ced.depth or ced.outputdim)"; + return false; + } + return true; } ggml_tensor* ModelLoader::tensor(const std::string& n) const { diff --git a/src/model_loader.hpp b/src/model_loader.hpp index 7c406bf..f27c5bb 100644 --- a/src/model_loader.hpp +++ b/src/model_loader.hpp @@ -38,6 +38,17 @@ class ModelLoader { ModelLoader() = default; ~ModelLoader(); bool load(const std::string& path); + // Load from a GGUF held in memory. Nothing is read from disk, and `data` is + // only read during the call: the tensor data is copied into the loader's own + // buffers, so the caller may free or reuse `data` afterwards. + // + // A non-empty `prefix` selects one model inside a larger GGUF (a bundle): + // every metadata key and tensor name of the model is stored as + // ``. Only the tensors under the prefix are copied, and they + // are seen under their unprefixed names. + bool load_from_memory(const void* data, size_t size, const std::string& prefix = ""); + // Why the last load failed ("" if it did not). + const std::string& error() const { return err_; } const CedConfig& config() const { return cfg_; } ggml_tensor* tensor(const std::string& name) const; // nullptr if absent // Make every weight usable as a graph leaf on `backend`. CPU: zero-copy @@ -51,7 +62,10 @@ class ModelLoader { const float* host_f32(const std::string& name) const; private: + bool read_model(const std::string& prefix); // shared by every load path + CedConfig cfg_; + std::string err_; gguf_context* gguf_ = nullptr; ggml_context* ctx_ = nullptr; ggml_context* dev_ctx_ = nullptr; // no_alloc mirror of ctx_ (GPU path) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ba9e679..0eacd6b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,7 +1,7 @@ set(CED_MODEL_GGUF "${PROJECT_SOURCE_DIR}/models/ced-base-f32.gguf") set(CED_BASELINE_GGUF "${PROJECT_SOURCE_DIR}/tests/fixtures/ced-base.baseline.gguf") -foreach(t blocks frontend quant capi) +foreach(t blocks frontend quant capi memory) add_executable(test_${t} test_${t}.cpp) target_link_libraries(test_${t} PRIVATE ced) target_include_directories(test_${t} PRIVATE @@ -9,6 +9,11 @@ foreach(t blocks frontend quant capi) ${PROJECT_SOURCE_DIR}/include ${PROJECT_SOURCE_DIR}/third_party/ggml/include) endforeach() +# Load from memory (plain and prefixed) vs the path loader, bit for bit, plus +# truncated/corrupt buffers. Needs two different CED models (tiny and mini). +add_test(NAME memory COMMAND test_memory + "${CMAKE_SOURCE_DIR}/models/ced-tiny-q8_0.gguf" + "${CMAKE_SOURCE_DIR}/models/ced-mini-q8_0.gguf") add_test(NAME capi COMMAND test_capi "${CED_MODEL_GGUF}" "${CED_BASELINE_GGUF}") # Per-component parity (f32 reference path). diff --git a/tests/test_memory.cpp b/tests/test_memory.cpp new file mode 100644 index 0000000..18e0b4e --- /dev/null +++ b/tests/test_memory.cpp @@ -0,0 +1,334 @@ +// Tests for loading a model from memory (ced_capi_load_from_memory and the +// prefixed variant). No baseline fixture is needed: the reference is the same +// model loaded from its path, and outputs must match bit for bit. +// +// argv[1] = model gguf (any CED size), argv[2] = a second, different model gguf +// (used as the other component of a bundle). +#include "ced_capi.h" + +#include "ggml.h" +#include "gguf.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +int g_fail = 0; +#define CHECK(cond, ...) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s -- ", __FILE__, __LINE__, #cond); \ + std::fprintf(stderr, __VA_ARGS__); \ + std::fprintf(stderr, "\n"); \ + ++g_fail; \ + } \ + } while (0) + +std::vector read_file(const std::string& path) { + std::ifstream f(path, std::ios::binary); + return std::vector((std::istreambuf_iterator(f)), std::istreambuf_iterator()); +} + +// Deterministic 3 s test clip: a few tones plus a noise floor. +std::vector make_wav() { + std::vector w(16000 * 3); + uint32_t s = 12345; + for (size_t i = 0; i < w.size(); ++i) { + s = s * 1664525u + 1013904223u; + float noise = ((s >> 8) / (float)(1 << 24) - 0.5f) * 0.05f; + float t = (float)i / 16000.0f; + w[i] = 0.3f * std::sin(6.2831853f * 440.0f * t) + 0.2f * std::sin(6.2831853f * 1250.0f * t * (1 + t)) + noise; + } + return w; +} + +bool scores(ced_ctx* c, const std::vector& wav, std::vector& out) { + int n = ced_capi_num_classes(c); + out.assign(n, 0.0f); + return ced_capi_classify_pcm_probs(c, wav.data(), (int)wav.size(), 16000, out.data(), n) == n; +} + +bool bitwise_equal(const std::vector& a, const std::vector& b) { + return a.size() == b.size() && std::memcmp(a.data(), b.data(), a.size() * sizeof(float)) == 0; +} + +// Copy `n` bytes of `src` into a buffer of exactly that size, so ASan flags any +// read past the end. +std::vector exact_copy(const std::vector& src, size_t n) { + return std::vector(src.begin(), src.begin() + n); +} + +// ---- in-memory bundle builder ------------------------------------------------ + +void copy_kv(gguf_context* out, gguf_context* g, int64_t i, const std::string& name) { + const char* k = name.c_str(); + switch (gguf_get_kv_type(g, i)) { + case GGUF_TYPE_UINT32: gguf_set_val_u32(out, k, gguf_get_val_u32(g, i)); break; + case GGUF_TYPE_FLOAT32: gguf_set_val_f32(out, k, gguf_get_val_f32(g, i)); break; + case GGUF_TYPE_BOOL: gguf_set_val_bool(out, k, gguf_get_val_bool(g, i)); break; + case GGUF_TYPE_STRING: gguf_set_val_str(out, k, gguf_get_val_str(g, i)); break; + case GGUF_TYPE_ARRAY: { + const gguf_type at = gguf_get_arr_type(g, i); + const size_t n = gguf_get_arr_n(g, i); + if (at == GGUF_TYPE_STRING) { + std::vector v(n); + for (size_t j = 0; j < n; ++j) v[j] = gguf_get_arr_str(g, i, j); + gguf_set_arr_str(out, k, v.data(), n); + } else { + gguf_set_arr_data(out, k, at, gguf_get_arr_data(g, i), n); + } + break; + } + default: break; + } +} + +struct Component { + std::string prefix; + const std::vector* gguf; + std::string retype_key; // if set, this key is written as a float instead +}; + +// Builds one GGUF holding every component, each under its prefix. Keys and +// tensors are renamed ``; tensor bytes are copied unchanged. +std::vector build_bundle(const std::vector& comps) { + gguf_context* out = gguf_init_empty(); + gguf_set_val_str(out, "general.architecture", "test-bundle"); + std::vector srcs; + std::vector ctxs; + ggml_context* tctx = ggml_init({ggml_tensor_overhead() * 4096, nullptr, true}); + std::vector datas; + for (auto& c : comps) { + ggml_context* sc = nullptr; + gguf_context* g = gguf_init_from_buffer(c.gguf->data(), c.gguf->size(), {false, &sc}); + if (!g) { std::fprintf(stderr, "bundle: bad source\n"); std::exit(2); } + srcs.push_back(g); + ctxs.push_back(sc); + for (int64_t i = 0; i < gguf_get_n_kv(g); ++i) { + std::string k = gguf_get_key(g, i); + if (k == "general.alignment") continue; + if (!c.retype_key.empty() && k == c.retype_key) gguf_set_val_f32(out, (c.prefix + k).c_str(), 1.0f); + else copy_kv(out, g, i, c.prefix + k); + } + for (ggml_tensor* t = ggml_get_first_tensor(sc); t; t = ggml_get_next_tensor(sc, t)) { + ggml_tensor* d = ggml_new_tensor(tctx, t->type, GGML_MAX_DIMS, t->ne); + ggml_set_name(d, (c.prefix + ggml_get_name(t)).c_str()); + d->data = t->data; + gguf_add_tensor(out, d); + datas.push_back(t->data); + } + } + const size_t meta = gguf_get_meta_size(out); + std::vector buf(meta, 0); + gguf_get_meta_data(out, buf.data()); + const size_t align = 32; + for (int64_t i = 0; i < gguf_get_n_tensors(out); ++i) { + const size_t off = meta + gguf_get_tensor_offset(out, i); + if (buf.size() < off) buf.resize(off, 0); + const uint8_t* p = (const uint8_t*)datas[i]; + buf.insert(buf.end(), p, p + gguf_get_tensor_size(out, i)); + buf.resize((buf.size() + align - 1) / align * align, 0); + } + for (auto g : srcs) gguf_free(g); + for (auto c : ctxs) ggml_free(c); + ggml_free(tctx); + gguf_free(out); + return buf; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc < 3) { + std::fprintf(stderr, "usage: %s model.gguf other-model.gguf\n", argv[0]); + return 2; + } + const std::string path = argv[1], path2 = argv[2]; + const std::vector file = read_file(path), file2 = read_file(path2); + CHECK(!file.empty() && !file2.empty(), "cannot read models"); + const std::vector wav = make_wav(); + + // Reference: path loader. + ced_ctx* ref = ced_capi_load(path.c_str()); + ced_ctx* ref2 = ced_capi_load(path2.c_str()); + if (!ref || !ref2) { std::fprintf(stderr, "FAIL: path load\n"); return 1; } + std::vector ref_s, ref_s2; + CHECK(scores(ref, wav, ref_s) && scores(ref2, wav, ref_s2), "reference classify"); + CHECK(!bitwise_equal(ref_s, ref_s2), "the two test models must differ"); + + // 1. Same file from memory: identical scores, labels and rate. + { + std::vector heap = file; + ced_ctx* m = ced_capi_load_from_memory(heap.data(), heap.size()); + CHECK(m != nullptr, "load_from_memory: %s", ced_capi_last_error(nullptr)); + if (m) { + // 2. The buffer is not used after the call: wipe and free it first. + std::memset(heap.data(), 0xA5, heap.size()); + std::vector().swap(heap); + std::vector s; + CHECK(scores(m, wav, s) && bitwise_equal(s, ref_s), "memory scores differ from path scores"); + CHECK(ced_capi_num_classes(m) == ced_capi_num_classes(ref), "num_classes"); + CHECK(ced_capi_sample_rate(m) == ced_capi_sample_rate(ref), "sample_rate"); + CHECK(std::strcmp(ced_capi_label(m, 0), ced_capi_label(ref, 0)) == 0, "label 0"); + ced_capi_free(m); + } + } + + // 3. Prefixed view of a two-component bundle, both orders. + for (int order = 0; order < 2; ++order) { + std::vector comps = order == 0 + ? std::vector{{"ced.", &file, ""}, {"other.", &file2, ""}} + : std::vector{{"other.", &file2, ""}, {"ced.", &file, ""}}; + std::vector bundle = build_bundle(comps); + ced_ctx* a = ced_capi_load_from_memory_prefixed(bundle.data(), bundle.size(), "ced."); + ced_ctx* b = ced_capi_load_from_memory_prefixed(bundle.data(), bundle.size(), "other."); + CHECK(a && b, "prefixed load (order %d): %s", order, ced_capi_last_error(nullptr)); + std::memset(bundle.data(), 0x5A, bundle.size()); + std::vector().swap(bundle); + if (a && b) { + std::vector sa, sb; + CHECK(scores(a, wav, sa) && bitwise_equal(sa, ref_s), "prefixed 'ced.' scores (order %d)", order); + CHECK(scores(b, wav, sb) && bitwise_equal(sb, ref_s2), "prefixed 'other.' scores (order %d)", order); + } + ced_capi_free(a); + ced_capi_free(b); + } + + // 4. Bad arguments and the wrong prefix. + CHECK(ced_capi_load_from_memory(nullptr, 100) == nullptr, "NULL data"); + CHECK(std::strlen(ced_capi_last_error(nullptr)) > 0, "error set for NULL data"); + CHECK(ced_capi_load_from_memory(file.data(), 0) == nullptr, "size 0"); + CHECK(ced_capi_load_from_memory_prefixed(file.data(), file.size(), nullptr) == nullptr, "NULL prefix"); + CHECK(ced_capi_load_from_memory_prefixed(file.data(), file.size(), "") == nullptr, "empty prefix"); + { + std::vector bundle = build_bundle({{"ced.", &file, ""}}); + CHECK(ced_capi_load_from_memory_prefixed(bundle.data(), bundle.size(), "nope.") == nullptr, "wrong prefix"); + CHECK(std::strstr(ced_capi_last_error(nullptr), "no tensors under prefix") != nullptr, "wrong prefix message: %s", + ced_capi_last_error(nullptr)); + // A standalone file has no prefix at all. + CHECK(ced_capi_load_from_memory_prefixed(file.data(), file.size(), "ced.") == nullptr, "prefix on standalone file"); + // Not a bundle: loading it without a prefix finds no model keys. + CHECK(ced_capi_load_from_memory(bundle.data(), bundle.size()) == nullptr, "bundle without prefix"); + CHECK(std::strstr(ced_capi_last_error(nullptr), "not a ced model") != nullptr, "bundle w/o prefix message: %s", + ced_capi_last_error(nullptr)); + } + + // 5. A key with the wrong type is an error, not an abort. + { + std::vector bad = build_bundle({{"ced.", &file, "ced.depth"}}); + CHECK(ced_capi_load_from_memory_prefixed(bad.data(), bad.size(), "ced.") == nullptr, "retyped key (prefixed)"); + CHECK(std::strstr(ced_capi_last_error(nullptr), "unexpected type") != nullptr, "message: %s", ced_capi_last_error(nullptr)); + std::vector bad2 = build_bundle({{"", &file, "ced.depth"}}); + CHECK(ced_capi_load_from_memory(bad2.data(), bad2.size()) == nullptr, "retyped key (plain)"); + } + + // 6. Truncation at every region: header (dense), tensor table, tensor data. + { + std::vector bundle = build_bundle({{"ced.", &file, ""}, {"other.", &file2, ""}}); + struct Case { const char* name; const std::vector* buf; const char* prefix; }; + const Case cases[] = {{"plain", &file, nullptr}, {"bundle", &bundle, "ced."}}; + for (const Case& cs : cases) { + const size_t total = cs.buf->size(); + std::vector cuts; + for (size_t n = 0; n < 512 && n < total; ++n) cuts.push_back(n); + for (size_t n = 512; n < 200000 && n < total; n += 257) cuts.push_back(n); + for (int i = 1; i < 120; ++i) cuts.push_back(total * i / 120); + cuts.push_back(total - 1); + int accepted = 0; + for (size_t n : cuts) { + std::vector t = exact_copy(*cs.buf, n); + ced_ctx* c = cs.prefix ? ced_capi_load_from_memory_prefixed(t.data(), t.size(), cs.prefix) + : ced_capi_load_from_memory(t.data(), t.size()); + if (c) { + ++accepted; + ced_capi_free(c); + } else { + CHECK(std::strlen(ced_capi_last_error(nullptr)) > 0, "%s: no error text at %zu", cs.name, n); + } + } + // A plain file cut anywhere must be rejected. A bundle view may succeed + // only once the cut is past the bytes of the selected component. + if (!cs.prefix) CHECK(accepted == 0, "%s: %d truncated buffers accepted", cs.name, accepted); + else std::fprintf(stderr, "[%s] %zu cuts, %d accepted (cut after the component)\n", cs.name, cuts.size(), accepted); + } + // Cut inside the selected component (first in the bundle): must be rejected. + std::vector first = build_bundle({{"ced.", &file, ""}, {"other.", &file2, ""}}); + std::vector t = exact_copy(first, first.size() / 10 + 100000 < first.size() ? first.size() / 10 + 100000 : first.size() / 2); + CHECK(ced_capi_load_from_memory_prefixed(t.data(), t.size(), "ced.") == nullptr, "cut inside component"); + CHECK(std::strstr(ced_capi_last_error(nullptr), "truncated") != nullptr || + std::strstr(ced_capi_last_error(nullptr), "corrupt") != nullptr, "cut message: %s", ced_capi_last_error(nullptr)); + } + + // 7. Corrupt headers. + { + auto expect_reject = [&](const char* what, std::vector b) { + std::vector t = exact_copy(b, b.size()); + CHECK(ced_capi_load_from_memory(t.data(), t.size()) == nullptr, "%s accepted", what); + CHECK(std::strlen(ced_capi_last_error(nullptr)) > 0, "%s: no error text", what); + }; + std::vector b = file; + b[0] ^= 0xff; expect_reject("bad magic", b); + b = file; b[4] = 0xff; b[5] = 0xff; expect_reject("bad version", b); + b = file; std::memset(&b[8], 0xff, 8); expect_reject("huge tensor count", b); + b = file; std::memset(&b[16], 0xff, 8); expect_reject("huge kv count", b); + b = file; std::memset(&b[8], 0, 8); expect_reject("zero tensors", b); + b.assign(file.size(), 0); expect_reject("all zero", b); + b.assign(file.size(), 0xff); expect_reject("all 0xff", b); + b.assign(8, 0); expect_reject("8 zero bytes", b); + } + + // 8. Bit flips in the header and tensor table must never crash. + { + uint32_t s = 777; + auto rnd = [&]() { s = s * 1664525u + 1013904223u; return s >> 8; }; + int ok = 0, rejected = 0; + const size_t span = std::min(file.size(), 4096); + for (int it = 0; it < 400; ++it) { + std::vector b = exact_copy(file, file.size()); + for (int k = 0; k < 1 + (int)(rnd() % 4); ++k) b[rnd() % span] ^= (uint8_t)(1u << (rnd() % 8)); + ced_ctx* c = ced_capi_load_from_memory(b.data(), b.size()); + if (c) { ++ok; ced_capi_free(c); } else ++rejected; + } + std::fprintf(stderr, "[bitflip] %d accepted, %d rejected, no crash\n", ok, rejected); + } + + // 9. Concurrent loads from memory (plain and prefixed) match the reference. + { + std::vector bundle = build_bundle({{"ced.", &file, ""}, {"other.", &file2, ""}}); + std::atomic bad{0}; + std::vector th; + for (int t = 0; t < 6; ++t) + th.emplace_back([&, t]() { + for (int i = 0; i < 3; ++i) { + ced_ctx* c; + const std::vector* want; + if (t % 3 == 0) { c = ced_capi_load_from_memory(file.data(), file.size()); want = &ref_s; } + else if (t % 3 == 1) { c = ced_capi_load_from_memory_prefixed(bundle.data(), bundle.size(), "ced."); want = &ref_s; } + else { c = ced_capi_load_from_memory_prefixed(bundle.data(), bundle.size(), "other."); want = &ref_s2; } + std::vector s; + if (!c || !scores(c, wav, s) || !bitwise_equal(s, *want)) ++bad; + ced_capi_free(c); + // Each thread has its own load error. + if (ced_capi_load_from_memory(nullptr, 0) != nullptr || std::strlen(ced_capi_last_error(nullptr)) == 0) ++bad; + } + }); + for (auto& x : th) x.join(); + CHECK(bad == 0, "%d concurrent failures", bad.load()); + } + + ced_capi_free(ref); + ced_capi_free(ref2); + std::fprintf(stderr, "%s\n", g_fail ? "FAIL" : "PASS"); + return g_fail ? 1 : 0; +}