diff --git a/README.md b/README.md index ceebfa7..3b7d720 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,8 @@ parakeet-cli transcribe ... --beam-size 4 --nbest 4 # TDT N-best, se parakeet-cli transcribe ... --stream # cache-aware streaming (EOU and Nemotron models) parakeet-cli transcribe ... --lang # Nemotron 3.5 language, default auto parakeet-cli transcribe ... --vad [--vad-model silero.gguf] # cut long audio at pauses (offline, greedy only) +parakeet-cli transcribe ... --vad-trim SEC # trim each piece to its speech plus SEC (default 0.3, 0 = whole cuts) +parakeet-cli transcribe ... --min-local-conf 0.5 # opt-in: drop words invented on noise (docs/vad.md) parakeet-cli vad --model M --input A.wav [--mode segments] [--probabilities] # speech regions as JSON parakeet-cli scene --model ASR --diar DIAR --sound CED --input A.wav # words + speakers + sounds parakeet-cli scene ... --speakers SPK.gguf --registry R # name the speakers diff --git a/docs/vad-benchmarks.md b/docs/vad-benchmarks.md index fd91aa5..059e52f 100644 --- a/docs/vad-benchmarks.md +++ b/docs/vad-benchmarks.md @@ -925,6 +925,89 @@ compared with the model runs, but it was not timed separately. Scripts, the small result files and how to reproduce: [`scripts/vad_bench/fusion/`](../scripts/vad_bench/fusion/README.md). +## Trimming segments and the word filter + +Two changes to what `transcribe --vad` hands to the decoder. Scripts and the full output are in +[scripts/vad_bench/decoder_guards](../scripts/vad_bench/decoder_guards) (`results/tables.txt`). The +options are described in [vad.md](vad.md). + +The run was not made on a quiet machine: the load average was between 5 and 50 (see the first line +of `results/tables.txt`). Word error rates and word counts do not depend on load; no timing is +claimed. + +### Setup + +- Old: `parakeet-cli` before the change (the cuts of the previous segmenter). New: the default, + each segment trimmed to its speech plus 0.3 s. `--vad-trim 0` gave the old output byte for byte on + all 45 files that were run both ways (15 per detector). +- Detectors: the Ultra head, the Redux head (dequantized F16), and TDT 0.6B v3 with Silero. + The model was Ultra F16 for the head runs. +- Talks: three whole TED-LIUM long-form talks that were not used to choose any setting (5627 + reference words). Words are lower-cased, punctuation removed. +- Speech in noise: 4 sets of 6 LibriSpeech test-clean utterances with gaps (about 50 s each, so the + segmenter cuts them), clean, with white noise at 5 dB SNR, or with pink noise at 0 dB SNR. 429 + reference words per condition. +- Noise block: 90 s of a talk with 60 s of synthetic noise (white, pink, clicks, music-like tones) put + in at a quiet point, 30 files. Any word the decoder returns inside the block is an invented word. + "Seconds decoded" is the overlap of the cuts with the block, from `vad --mode segments`. +- Noise alone: 63 files of 30 s (seven noise types, three levels), decoded whole. + +### Trimming: word error rate + +| Set | Detector | Old | Trim 0.3 | Change | +| --- | --- | ---: | ---: | ---: | +| 3 talks | Ultra head | 3.68 | 3.68 | +0.00 | +| 3 talks | Redux head | 4.39 | 4.51 | +0.12 | +| 3 talks | v3 + Silero | 3.45 | 3.45 | +0.00 | +| clean | Ultra / Redux / v3 + Silero | 2.10 / 3.26 / 1.86 | 1.86 / 3.03 / 1.86 | -0.23 / -0.23 / +0.00 | +| white noise 5 dB | Ultra / Redux / v3 + Silero | 4.90 / 7.23 / 5.59 | 4.43 / 6.76 / 5.13 | -0.47 / -0.47 / -0.47 | +| pink noise 0 dB | Ultra / Redux / v3 + Silero | 6.53 / 12.35 / 7.93 | 5.59 / 13.29 / 7.69 | -0.93 / +0.93 / -0.23 | + +On talks the change is within 0.12 points; the Redux head loses a little on two of the three talks +and on pink noise. The speech-in-noise sets are small (one word is 0.23 points), so read them as +"neutral", not as a gain. + +### Trimming: the noise block + +| Detector | Seconds of the 60 s block decoded, old | New | Invented words, old | New | +| --- | ---: | ---: | ---: | ---: | +| Ultra head | 33.8 | 12.0 | 0 | 0 | +| Redux head | 29.7 | 0.3 | 0 | 0 | +| v3 + Silero | 27.7 | 0.1 | 11 (5 files) | 0 | + +The seconds are means over the 30 files. The head still passes loud white and pink noise (-5 dB +against the speech) as speech, and then the trim cannot remove it: Ultra decodes 52 to 58 s of +those blocks. The Redux head also fires on digital silence. The words the decoder invented were +rare in this run (Ultra and Redux gave none, in the block or in 63 noise-only files), so the +trim's benefit here is mostly seconds saved, and for v3 the 11 invented words. + +### Word filter at 0.5 + +| Check | Ultra head | v3 + Silero | +| --- | ---: | ---: | +| WER on the 3 talks, trim 0.3 / plus filter | 3.68 / 3.68 | 3.45 / 3.45 | +| WER on speech in noise (12 files, 1287 words): off / 0.5 / 0.7 / 0.9 | 3.96 / 3.96 / 3.96 / 6.06 | 4.90 / 4.90 / 4.90 / 16.08 | +| Words dropped at 0.9 on those files | 31 | 164 | +| Words dropped at 0.5 on all files with real speech | 0 of 15375 | 3 of 15406 | +| Invented words in the 5 blocks that had them (old cuts, filter alone) | no events | 11 -> 1, 0 of 1484 other words lost | +| Invented words in the 63 noise-only files | 0 | 1 -> 0 (v3) | + +At 0.5 the filter is free on these sets and it removes most of the few invented words there were. +At 0.9 it costs real words, most on v3. Hence 0.5 is the suggested value. + +### Limits + +- The noise is synthetic. The event counts are small: 11 invented words in 5 files, all from v3. A + filter that removes 10 of 11 is a weak estimate of a rate. +- The filter was not run against invented words on Ultra, Redux or an RNN-T model (none occurred), and + not at all on a CTC model of 0.6B (a unit test runs it on one fixture). The `drop_punct_only` + option for CTC rests on the 110M CTC head, not measured here. +- A single lone invented word between real speech was not tested. +- Silero followed by the head (the two-stage rule), GPU backends and the quantized files other than the + F16 ones were not tested. +- The trim changes transcripts of long audio through the VAD paths slightly; the numbers above are + from three talks and 12 clips per condition. + ## When to use which This is limited to what the numbers above support. @@ -977,6 +1060,7 @@ are committed. In short: `silero_collect.py`, `silero_analyze.py`; speed with `speed_silero.py`. 5. Long talks: `longform_b1.sh`. 6. Noise root-cause study: [`noise_dive/`](../scripts/vad_bench/noise_dive/README.md). +7. Segment trim and word filter: [`decoder_guards/`](../scripts/vad_bench/decoder_guards/README.md). Small result files of the runs on this page (tables, per run timings and load logs) are in `scripts/vad_bench/results/`. The raw per clip predictions are not committed; diff --git a/docs/vad.md b/docs/vad.md index 5388a08..c7da89d 100644 --- a/docs/vad.md +++ b/docs/vad.md @@ -98,6 +98,7 @@ the same keys). Unknown keys and out of range values are errors. | `min_speech` | seconds; shorter speech runs are dropped | 0.1 | 0.25 | | `speech_pad` | seconds >= 0, `speech` mode: widen each region on both sides | 0 | 0.03 | | `max_segment` | seconds; cap in `segments` mode | 30 | 30 | +| `trim` | seconds >= 0; `segments` mode and the transcribe functions: shrink each cut to its speech plus this much on each side, 0 = keep the whole cut | 0.3 | 0.3 | | `mode` | `speech` or `segments` | `speech` | `speech` | | `probabilities` | add the per frame probabilities | false | false | @@ -106,7 +107,64 @@ shorter than 0.1 s are bridged, runs shorter than `min_speech` are dropped, regions closer than `min_pause` merge, then each region is padded (and two regions that would overlap meet in the middle of the gap). `segments` is the cut that `transcribe --vad` decodes: pieces of at most `max_segment` seconds cut at -pauses, pieces without speech dropped, audio within the cap returned whole. +pauses, pieces without speech dropped, audio within the cap returned whole. Each +kept piece is then trimmed (see below). + +### Trimming the segments + +A piece cut from long audio used to carry everything between its two cut points +to the decoder, including the noise and silence the VAD had already flagged as +non-speech. That is where an ASR model invents words. Since this change each +piece shrinks to its first speech frame minus `trim` and its last speech frame +plus `trim` (default 0.3 s, never past the cut itself). Speech is the smoothed +mask, so the rules above still decide what counts as speech. Audio within the +cap is not cut and not trimmed. Word and token times are still relative to the +whole file. `trim` 0 (`--vad-trim 0`) gives the previous cuts exactly. + +This is a change of default behaviour for `transcribe --vad`, +`parakeet_capi_transcribe_path_json_vad*` and the `segments` mode of the VAD +functions, for the head, for Silero and for VAD-only slices (they share the +segmenter). On talks, transcripts of long audio can shift slightly (a word WER +cost of about 0.1 point in our runs); on audio with long noisy stretches the +decoder sees much less noise. Numbers: [vad-benchmarks.md](vad-benchmarks.md#trimming-segments-and-the-word-filter). + +## Word filter (opt-in) + +A confidence filter can remove the words a model invents on noise. It is +post-processing of the decode: the model, the VAD and the cuts are unchanged, and +it is off by default (output is then byte for byte the same). + +A word is dropped when the mean confidence of the words that start within +`local_radius` seconds of it (the word itself included, the same decode unit) is +below `min_local_conf`. A low confidence word between confident ones keeps a high +mean and stays; a word that stands alone, or among other low confidence words, +goes. A decode unit is the whole clip, or one VAD segment. `drop_punct_only` also +removes words that consist only of punctuation. + +| Option | Meaning | Default | +| --- | --- | --- | +| `min_local_conf` | 0 to 1; 0 = off. 0.5 is the suggested value | 0 | +| `local_radius` | seconds, both sides | 5 | +| `drop_punct_only` | drop words with no letter or digit; recommended for CTC models | false | + +``` +parakeet-cli transcribe --model m.gguf --input a.wav --min-local-conf 0.5 \ + [--local-radius SEC] [--drop-punct-only] [--json] [--vad ...] + +char* parakeet_capi_transcribe_path_json_with(parakeet_ctx* ctx, const char* wav_path, + int decoder, const char* options_json); +``` + +The options JSON of `parakeet_capi_transcribe_path_json_with` takes the three +keys above. `parakeet_capi_transcribe_path_json_vad_with` takes them too, next to +the VAD keys (`trim` included). With a filter on, the JSON document gets one more +member, `"guard":{"dropped_words":N}` (N is 0 when nothing was dropped), and the +dropped words are also removed from `text`, `words` and `tokens`. + +Limits: 0.5 removes hallucinated words on Ultra, Redux and RNN-T models at no +cost in WER on clean speech. On v3 and on CTC models, and at higher thresholds, +it also removes real words (see the benchmark page). It does not save time: the +decoder still runs on everything it is given. ### Why the Silero defaults differ @@ -167,7 +225,8 @@ audio with Silero: ``` parakeet-cli transcribe --model tdt-0.6b-v3.gguf --input long.wav --vad --vad-model silero.gguf \ - [--vad-threshold F] [--vad-min-pause SEC] [--vad-min-speech SEC] [--vad-max-seg SEC] + [--vad-threshold F] [--vad-min-pause SEC] [--vad-min-speech SEC] [--vad-max-seg SEC] \ + [--vad-trim SEC] char* parakeet_capi_transcribe_path_json_vad_with(parakeet_ctx* asr, parakeet_ctx* silero, const char* wav_path, int decoder, diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 7ebc497..c60a46a 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -373,8 +373,9 @@ static int cmd_transcribe_stream(const std::string& model, const std::string& in // Segmenter options the user set on the command line. A value left unset keeps // the default of the VAD in use (Ultra/Redux head or Silero). struct VadOverrides { - std::optional threshold, min_pause, min_speech, max_seg, pad; + std::optional threshold, min_pause, min_speech, max_seg, pad, trim; void apply(pk::SegmenterOpts& o) const { + if (trim) o.trim_sec = *trim; if (threshold) o.threshold = (float)*threshold; if (min_pause) o.min_pause_sec = *min_pause; if (min_speech) o.min_speech_sec = *min_speech; @@ -386,7 +387,7 @@ struct VadOverrides { static int cmd_transcribe_vad(const std::string& model, const std::string& input, pk::Decoder dec, const std::string& lang, bool timestamps, bool json, const VadOverrides& ov, const std::string& vad_model, - const std::string& vad_component) { + const std::string& vad_component, const pk::WordFilter& wf) { pk::Audio audio; if (!load_audio_arg_16k_mono(input, audio)) { std::fprintf(stderr, "parakeet-cli: failed to load audio %s\n", input.c_str()); @@ -433,7 +434,7 @@ static int cmd_transcribe_vad(const std::string& model, const std::string& input const pk::Model::VadProbabilityFn* ext = silero ? &fn : nullptr; if (json || timestamps) { pk::Transcription tr = - m->transcribe_pcm_vad_with_timestamps(audio.samples, audio.sample_rate, dec, lang, opts, ext); + m->transcribe_pcm_vad_with_timestamps(audio.samples, audio.sample_rate, dec, lang, opts, ext, wf); if (json) { std::printf("%s\n", pk::transcription_to_json(tr, model_frame_sec(*m)).c_str()); } else { @@ -441,7 +442,7 @@ static int cmd_transcribe_vad(const std::string& model, const std::string& input std::printf("%.2f-%.2f %s (%.2f)\n", w.start, w.end, w.text.c_str(), w.conf); } } else { - std::printf("%s\n", m->transcribe_pcm_vad(audio.samples, audio.sample_rate, dec, lang, opts, ext).c_str()); + std::printf("%s\n", m->transcribe_pcm_vad(audio.samples, audio.sample_rate, dec, lang, opts, ext, wf).c_str()); } } catch (const std::exception& e) { std::fprintf(stderr, "parakeet-cli: transcribe failed: %s\n", e.what()); @@ -463,12 +464,18 @@ static int cmd_transcribe(int argc, char** argv) { bool vad = false; std::string vad_model, vad_component, component; VadOverrides vad_ov; + pk::WordFilter word_filter; double d = 0.0; auto parse_pos = [](const char* str, double& out) { char* end = nullptr; out = std::strtod(str, &end); return end != str && *end == '\0' && std::isfinite(out) && out > 0.0 && out <= 1e6; }; + auto parse_nonneg = [](const char* str, double& out) { + char* end = nullptr; + out = std::strtod(str, &end); + return end != str && *end == '\0' && std::isfinite(out) && out >= 0.0 && out <= 1e6; + }; bool score_norm = true; int beam_size = 0; int nbest = 0; @@ -515,6 +522,17 @@ static int cmd_transcribe(int argc, char** argv) { } else if (std::strcmp(argv[i], "--vad-min-speech") == 0 && i + 1 < argc) { if (!parse_pos(argv[++i], d)) { std::fprintf(stderr, "parakeet-cli: --vad-min-speech must be > 0\n"); return 2; } vad_ov.min_speech = d; + } else if (std::strcmp(argv[i], "--vad-trim") == 0 && i + 1 < argc) { + if (!parse_nonneg(argv[++i], d)) { std::fprintf(stderr, "parakeet-cli: --vad-trim must be >= 0 (0 = keep the whole cuts)\n"); return 2; } + vad_ov.trim = d; + } else if (std::strcmp(argv[i], "--min-local-conf") == 0 && i + 1 < argc) { + if (!parse_nonneg(argv[++i], d) || d > 1.0) { std::fprintf(stderr, "parakeet-cli: --min-local-conf must be in [0,1] (0 = off)\n"); return 2; } + word_filter.min_local_conf = (float)d; + } else if (std::strcmp(argv[i], "--local-radius") == 0 && i + 1 < argc) { + if (!parse_pos(argv[++i], d)) { std::fprintf(stderr, "parakeet-cli: --local-radius must be > 0\n"); return 2; } + word_filter.local_radius_sec = (float)d; + } else if (std::strcmp(argv[i], "--drop-punct-only") == 0) { + word_filter.drop_punct_only = true; } else if (std::strcmp(argv[i], "--vad-max-seg") == 0 && i + 1 < argc) { if (!parse_pos(argv[++i], d)) { std::fprintf(stderr, "parakeet-cli: --vad-max-seg must be > 0\n"); return 2; } vad_ov.max_seg = d; @@ -527,7 +545,8 @@ static int cmd_transcribe(int argc, char** argv) { "[--threads N] [--json] " "[--component NAME] " "[--vad [--vad-model ] [--vad-component NAME] [--vad-threshold F=0.5] [--vad-min-pause SEC] " - "[--vad-min-speech SEC] [--vad-max-seg SEC=30]] " + "[--vad-min-speech SEC] [--vad-max-seg SEC=30] [--vad-trim SEC=0.3]] " + "[--min-local-conf F [--local-radius SEC=5]] [--drop-punct-only] " "[--beam-size N [--nbest N] [--no-score-norm]]\n"); return 2; } @@ -538,6 +557,10 @@ static int cmd_transcribe(int argc, char** argv) { if (threads > 0) pk::set_num_threads(threads); if (stream) { + if (word_filter.active()) { + std::fprintf(stderr, "parakeet-cli: the word filter is offline only\n"); + return 2; + } if (vad) { std::fprintf(stderr, "parakeet-cli: --vad is offline only\n"); return 2; @@ -581,7 +604,39 @@ static int cmd_transcribe(int argc, char** argv) { std::fprintf(stderr, "parakeet-cli: --vad works with greedy decoding only\n"); return 2; } - return cmd_transcribe_vad(model, input, dec, lang, timestamps, json, vad_ov, vad_model, vad_component); + return cmd_transcribe_vad(model, input, dec, lang, timestamps, json, vad_ov, vad_model, vad_component, word_filter); + } + if (word_filter.active() && (beam_size != 0 || nbest != 0)) { + std::fprintf(stderr, "parakeet-cli: the word filter works with greedy decoding only\n"); + return 2; + } + if (word_filter.active()) { + // Greedy decode with timestamps, then the word filter on the whole clip. + pk::Audio audio; + if (is_stdin_input(input) && !load_audio_arg_16k_mono(input, audio)) { + std::fprintf(stderr, "parakeet-cli: failed to load audio stdin\n"); + return 1; + } + try { + std::unique_ptr m = load_asr(model); + if (!m) { std::fprintf(stderr, "parakeet-cli: failed to load model %s\n", model.c_str()); return 1; } + pk::Transcription tr = is_stdin_input(input) + ? m->transcribe_with_timestamps(audio.samples, audio.sample_rate, dec, lang) + : m->transcribe_path_with_timestamps(input, dec, lang); + pk::apply_word_filter(tr, word_filter); + if (json) { + std::printf("%s\n", pk::transcription_to_json(tr, model_frame_sec(*m)).c_str()); + } else if (timestamps) { + for (const pk::Word& w : tr.words) + std::printf("%.2f-%.2f %s (%.2f)\n", w.start, w.end, w.text.c_str(), w.conf); + } else { + std::printf("%s\n", tr.text.c_str()); + } + } catch (const std::exception& e) { + std::fprintf(stderr, "parakeet-cli: transcribe failed: %s\n", e.what()); + return 1; + } + return 0; } if (nbest != 0 && beam_size == 0) { std::fprintf(stderr, @@ -2245,6 +2300,9 @@ static int cmd_vad(int argc, char** argv) { } else if (std::strcmp(argv[i], "--max-segment") == 0 && i + 1 < argc) { if (!num(argv[++i], d, false)) return bad("--max-segment must be > 0"); ov.max_seg = d; + } else if (std::strcmp(argv[i], "--trim") == 0 && i + 1 < argc) { + if (!num(argv[++i], d, true)) return bad("--trim must be >= 0"); + ov.trim = d; } else if (std::strcmp(argv[i], "--mode") == 0 && i + 1 < argc) { const char* v = argv[++i]; if (std::strcmp(v, "speech") == 0) mode = pk::VadRequest::Mode::kSpeech; @@ -2258,7 +2316,7 @@ static int cmd_vad(int argc, char** argv) { std::fprintf(stderr, "usage: parakeet-cli vad --model --input " "[--component NAME] [--threshold F=0.5] [--min-pause SEC] [--min-speech SEC] [--speech-pad SEC] " - "[--max-segment SEC=30] [--mode speech|segments] [--probabilities] [--threads N]\n"); + "[--max-segment SEC=30] [--trim SEC=0.3] [--mode speech|segments] [--probabilities] [--threads N]\n"); return 2; } if (threads > 0) pk::set_num_threads(threads); diff --git a/include/parakeet_capi.h b/include/parakeet_capi.h index 4640853..69ebe9d 100644 --- a/include/parakeet_capi.h +++ b/include/parakeet_capi.h @@ -74,6 +74,8 @@ typedef struct parakeet_ctx parakeet_ctx; // parakeet_capi_transcribe_path_json_vad*) and Silero VAD contexts are // additive and keep ABI v10: a caller that needs them checks for the symbols // (dlsym) or for PARAKEET_MODEL_KIND_VAD. +// Word filter (parakeet_capi_transcribe_path_json_with and the filter keys of +// parakeet_capi_transcribe_path_json_vad_with) is additive and keeps ABI v10. // Bundle GGUF (parakeet_capi_load_component, parakeet_capi_bundle_components_json, // parakeet_capi_load_error; docs/bundle.md) is additive and keeps ABI v10. int parakeet_capi_abi_version(void); @@ -199,13 +201,37 @@ int parakeet_capi_transcribe_pcm_batch_lang(parakeet_ctx* ctx, char* parakeet_capi_transcribe_path_json(parakeet_ctx* ctx, const char* wav_path, int decoder); +// Like parakeet_capi_transcribe_path_json, with the optional word filter +// (additive; no ABI bump). `options_json` is a flat JSON object, or NULL / "" for +// no filter, in which case the document is the one of the plain function. +// Keys: +// "min_local_conf" 0 to 1; 0 = off (default). A word is dropped when the mean +// confidence of the words that start within "local_radius" +// seconds of it (itself included) is below this value. It +// removes words that stand alone or sit among other low +// confidence words, as noise tends to give, and keeps a +// doubtful word between confident ones. 0.5 is a good +// start. Higher values also drop real words on some models +// (see docs/vad-benchmarks.md). +// "local_radius" seconds > 0; default 5. +// "drop_punct_only" true to drop words that are only punctuation (a CTC model +// can emit a lone "." on noise); default false. +// Unknown keys and bad values are errors (NULL, last error set). The filter only +// post-processes the decode: with it off, or when it drops nothing, the words and +// tokens are the same as without it. When a filter is on, the document gets one +// more member, "guard":{"dropped_words":N}, with N = 0 when nothing was dropped; +// the dropped words are also removed from "text", "words" and "tokens". The +// filter works on the whole file as one decode unit. +char* parakeet_capi_transcribe_path_json_with(parakeet_ctx* ctx, const char* wav_path, + int decoder, const char* options_json); + // Like parakeet_capi_transcribe_path_json, but long audio is cut at pauses found // by the model's own VAD head into segments of at most 30 s, and the segments are // transcribed one by one (word/token times are relative to the whole file). Audio // of 30 s or less gives the same document as the plain function. Returns NULL and // sets the context's last error to "model has no VAD head" when the model has no // VAD head. It always uses the default segmenter options (30 s cap, threshold -// 0.5). Additive; no ABI bump. +// 0.5, each segment trimmed to its speech plus 0.3 s). Additive; no ABI bump. char* parakeet_capi_transcribe_path_json_vad(parakeet_ctx* ctx, const char* wav_path, int decoder); @@ -237,6 +263,9 @@ char* parakeet_capi_transcribe_path_json_vad(parakeet_ctx* ctx, const char* wav_ // "speech_pad" seconds >= 0; "speech" mode: each region is widened by this // on both sides; default 0 (Silero: 0.03) // "max_segment" seconds; segment cap in "segments" mode; default 30 +// "trim" seconds >= 0; "segments" mode: each segment shrinks to its +// first and last speech frame plus this much; default 0.3; +// 0 keeps the whole cuts // "mode" "speech" (default) or "segments" // "probabilities" true to add the per-frame probabilities; default false // Unknown keys and out-of-range values are errors. The Silero defaults are the @@ -277,8 +306,11 @@ char* parakeet_capi_vad_path_json(parakeet_ctx* ctx, const char* wav_path, // Silero context, or NULL to use the ASR model's own head (then the result is // as parakeet_capi_transcribe_path_json_vad, with the options below). Options // are the JSON object of parakeet_capi_vad_pcm_json; only "threshold", -// "min_pause", "min_speech" and "max_segment" are used here (the other keys are -// accepted and ignored). NULL or "" gives the defaults of the VAD in use. +// "min_pause", "min_speech", "max_segment" and "trim" are used here (the other +// keys of that object are accepted and ignored), plus the word filter keys of +// parakeet_capi_transcribe_path_json_with, which apply to each segment on its +// own (and to the whole file when it is at most max_segment seconds). NULL or +// "" gives the defaults of the VAD in use. // Audio of at most max_segment seconds (30 by default) is transcribed whole, // without running the VAD. Word and token times are relative to the whole file. // Returns the same document as parakeet_capi_transcribe_path_json. Errors set diff --git a/scripts/vad_bench/decoder_guards/README.md b/scripts/vad_bench/decoder_guards/README.md new file mode 100644 index 0000000..9fa8198 --- /dev/null +++ b/scripts/vad_bench/decoder_guards/README.md @@ -0,0 +1,38 @@ +# Trim and word filter runs + +Scripts behind the section "Trimming segments and the word filter" of +[docs/vad-benchmarks.md](../../../docs/vad-benchmarks.md). No audio and no model files are +committed. `results/` holds the output of the run on that page. + +`CLI_OLD` is the build of master at the commit before this change (91b120b), `CLI_NEW` the build of +the change. Python packages: `numpy soundfile librosa datasets`. Models: the Ultra and Redux GGUFs +(`moondream/parakeet-ultra`, `moondream/parakeet-redux`, F16 and dequantized), a TDT 0.6B v3 F16 +GGUF, and a Silero GGUF (`scripts/convert_silero_vad_to_gguf.py`). + +``` +python3 fetch_data.py DATA # 3 TED-LIUM talks (not the first four) and LibriSpeech test-clean +python3 make_corpus.py DATA # inserts of synthetic noise, speech in noise, manifest.json +python3 run_all.py DATA OUT --old CLI_OLD --new CLI_NEW --ultra ultra-f16.gguf \ + --redux redux-deq-f16.gguf --v3 tdt-0.6b-v3-f16.gguf --silero silero-vad-f16.gguf -j 3 +python3 tables.py DATA OUT > results/tables.txt +rm -r DATA # audio +``` + +`fetch_data.py` streams the data; nothing else is downloaded. `make_corpus.py` uses fixed seeds, +so the files are the same on every run. `run_all.py` needs a few hours of CPU on a loaded machine +and resumes when you start it again. It is not a timing run: the machine was loaded (the load +average is in `results/tables.txt`). + +What the files are: + +- `talk_*`: three whole talks, with the reference text of the dataset. +- `ins__`: 90 s of a talk with 60 s of synthetic noise (white, pink, clicks, + music-like tones) inserted at a quiet point, at -20 or -35 dB against the speech. The noise block is + where every word is an invented word. +- `sn__`: six LibriSpeech utterances with gaps, clean, with white noise at 5 dB SNR or + with pink noise at 0 dB SNR. + +`tables.py` prints, for each detector (Ultra head, Redux head, v3 with Silero): the check that +`--vad-trim 0` gives the old output byte for byte, the WER with the old cuts, with trim 0.3 and with +trim 0.3 plus `--min-local-conf 0.5`, the seconds of the noise block that the decoder gets, and the +words it returns inside the block. diff --git a/scripts/vad_bench/decoder_guards/fetch_data.py b/scripts/vad_bench/decoder_guards/fetch_data.py new file mode 100644 index 0000000..fbdb676 --- /dev/null +++ b/scripts/vad_bench/decoder_guards/fetch_data.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""usage: fetch_data.py OUT [--skip K] [--talks N] [--libri N] + +Streams the data of the trim and word filter runs; nothing is kept but what is written to OUT: + OUT/talk_.wav, talk_.txt TED-LIUM long-form (test), talks under 1500 s, after the + first K talks (default 4: the talks that were used to tune + things earlier are skipped), N talks (default 3) + OUT/libri.npy, libri.json every 13th LibriSpeech test-clean utterance, first N (default 60) +Delete OUT after the runs: it is audio.""" +import argparse, io, json, os, re +import numpy as np, soundfile as sf, librosa +from datasets import Audio, load_dataset + +ap = argparse.ArgumentParser() +ap.add_argument("out"); ap.add_argument("--skip", type=int, default=4) +ap.add_argument("--talks", type=int, default=3); ap.add_argument("--libri", type=int, default=60) +a = ap.parse_args(); os.makedirs(a.out, exist_ok=True) + +ds = load_dataset("distil-whisper/tedlium-long-form", split="test", streaming=True).cast_column("audio", Audio(decode=False)) +kept = 0 +for i, ex in enumerate(ds): + if i < a.skip: continue + y, sr = sf.read(io.BytesIO(ex["audio"]["bytes"]), dtype="float32") + if y.ndim > 1: y = y.mean(axis=1) + if sr != 16000: y = librosa.resample(y, orig_sr=sr, target_sr=16000) + if len(y) / 16000 > 1500: continue + name = re.sub(r"[^A-Za-z0-9_-]", "", os.path.basename(ex["audio"]["path"] or f"talk{i}").rsplit(".", 1)[0]) or f"talk{i}" + sf.write(f"{a.out}/talk_{name}.wav", y, 16000, subtype="PCM_16") + open(f"{a.out}/talk_{name}.txt", "w").write(" ".join(re.sub(r"<[^>]*>", " ", ex["text"]).split()) + "\n") + print("talk", name, round(len(y) / 16000), flush=True); kept += 1 + if kept >= a.talks: break + +ds = load_dataset("openslr/librispeech_asr", "clean", split="test", streaming=True).cast_column("audio", Audio(decode=False)) +U, T = [], [] +for i, ex in enumerate(ds): + if i % 13: continue + y, sr = sf.read(io.BytesIO(ex["audio"]["bytes"]), dtype="int16"); assert sr == 16000 + U.append(y); T.append(ex["text"].lower()) + if len(U) >= a.libri: break +np.save(f"{a.out}/libri.npy", np.array(U, dtype=object), allow_pickle=True) +json.dump(T, open(f"{a.out}/libri.json", "w")) +print("libri", len(U), flush=True) diff --git a/scripts/vad_bench/decoder_guards/make_corpus.py b/scripts/vad_bench/decoder_guards/make_corpus.py new file mode 100644 index 0000000..7f8f8a8 --- /dev/null +++ b/scripts/vad_bench/decoder_guards/make_corpus.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""usage: make_corpus.py DATA + +Builds, from the output of fetch_data.py, the files of the runs (all 16 kHz mono, > 30 s so that the +segmenter cuts them) and DATA/manifest.json: + talk_* the talks as they are (kind "talk", reference text) + ins__ 90 s of a talk with 60 s of synthetic noise inserted at a quiet point + (kind "insert", "ins": [start, end] of the noise block). The noise level is + `rel` dB against the RMS of the speech. Noise: white, pink, clicks, music-like tones. + nz__ 30 s of noise alone (white, pink, clicks, music, hum, tone, sweep), no speech + (kind "noise"; shorter than the 30 s cap, so decoded whole) + sn__ 6 LibriSpeech utterances with gaps (kind "sinr"), clean or with white or pink noise at an SNR + (white5 = white noise at 5 dB, pink0 = pink noise at 0 dB) +Seeds are fixed, so the files are the same on every run.""" +import glob, json, os, sys +import numpy as np, soundfile as sf + +D = sys.argv[1]; SR = 16000 +man = [] + +def rms(x): return float(np.sqrt((x ** 2).mean()) + 1e-12) +def at_db(x, db): return x * (10 ** (db / 20) / rms(x)) +def pink(n, r): + X = np.fft.rfft(r.standard_normal(n)); f = np.arange(len(X)); f[0] = 1 + y = np.fft.irfft(X / np.sqrt(f), n); return y / y.std() +def clicks(n, r): + x = np.zeros(n); t = 0 + while t < n - 400: + k = np.arange(400); x[t:t + 400] += r.uniform(0.3, 1) * r.choice([-1, 1]) * np.exp(-k / 40) * np.sin(k * 0.8) + t += int(r.uniform(0.15, 0.5) * SR) + return x +def music(n, r): + t = np.arange(n) / SR; x = np.zeros(n) + for f0 in (220.0, 261.63, 329.63, 440.0): + f = f0 * (1 + 0.003 * np.sin(2 * np.pi * r.uniform(4, 6) * t + r.uniform(0, 6))) + ph = 2 * np.pi * np.cumsum(f) / SR + for h, a in ((1, 1.0), (2, 0.5), (3, 0.25)): x += a * np.sin(h * ph) + env = 0.6 + 0.4 * np.sin(2 * np.pi * 0.5 * t + r.uniform(0, 6)) ** 2 + return x * env +def hum(n, r): + t = np.arange(n) / SR; f = r.choice([50.0, 60.0]) + return sum(np.sin(2 * np.pi * f * h * t + r.uniform(0, 6)) / h for h in (1, 2, 3, 5)) +def tone(n, r): return np.sin(2 * np.pi * float(r.choice([440, 1000, 2000])) * np.arange(n) / SR) +def sweep(n, r): + t = np.arange(n) / SR; T = t[-1]; return np.sin(2 * np.pi * (50 * t + (7500 - 50) * t * t / (2 * T))) +GEN = {"white": lambda n, r: r.standard_normal(n), "pink": pink, "clicks": clicks, "music": music, + "hum": hum, "tone": tone, "sweep": sweep} + +# talks and inserts +for wav in sorted(glob.glob(f"{D}/talk_*.wav")): + name = os.path.basename(wav)[5:-4] + x, sr = sf.read(wav, dtype="float32"); assert sr == SR + man.append(dict(name=f"talk_{name}", kind="talk", talk=name, ref=open(wav[:-4] + ".txt").read().strip())) + ex = x[60 * SR:150 * SR]; db = 20 * np.log10(rms(ex)) + fr = len(ex) // 1600; e = 10 * np.log10((ex[:fr * 1600].reshape(fr, 1600) ** 2).mean(1) + 1e-12) + c = [(e[i:i + 10].mean(), i) for i in range(250, 650)]; _, i0 = min(c); cut = (i0 + 5) * 1600 + for ti, (tn, rel) in enumerate([("white", -20), ("pink", -20), ("clicks", -20), ("music", -20), ("white", -35), ("music", -35), + ("white", -5), ("pink", -5), ("clicks", -5), ("music", -5)]): + r = np.random.default_rng(1000 + 17 * ti + len(name)) + n = np.clip(at_db(GEN[tn](60 * SR, r), db + rel), -1, 1) + y = np.concatenate([ex[:cut], n, ex[cut:]]).astype(np.float32) + nm = f"ins_{name}_{tn}{rel}" + sf.write(f"{D}/{nm}.wav", y, SR, subtype="PCM_16") + man.append(dict(name=nm, kind="insert", talk=name, noise=tn, level=rel, ins=[cut / SR, cut / SR + 60])) + + # 30 s of noise alone, at the same levels against the speech (decoded whole, no cuts) + for ni, tn in enumerate(["white", "pink", "clicks", "music", "hum", "tone", "sweep"]): + for rel in (-35, -20, -5): + r = np.random.default_rng(5000 + 100 * ni + abs(rel) + len(name)) + nm = f"nz_{name}_{tn}{rel}" + sf.write(f"{D}/{nm}.wav", np.clip(at_db(GEN[tn](30 * SR, r), db + rel), -1, 1).astype(np.float32), SR, subtype="PCM_16") + man.append(dict(name=nm, kind="noise", talk=name, noise=tn, level=rel)) + +# speech in noise from LibriSpeech +U = [u.astype(np.float32) / 32768 for u in np.load(f"{D}/libri.npy", allow_pickle=True)] +T = json.load(open(f"{D}/libri.json")) +def trim(u): + fr = len(u) // 160; r = np.sqrt((u[:fr * 160].reshape(fr, 160) ** 2).mean(1)) + idx = np.where(r > r.max() * 0.01)[0]; return u[idx[0] * 160:(idx[-1] + 1) * 160] +for k in range(len(U) // 6): + ids = list(range(6 * k, 6 * k + 6)); r = np.random.default_rng(2000 + k) + parts, spans, t = [r.standard_normal(int(r.uniform(.5, 1.5) * SR)) * 1e-3], [], 0.0 + t = len(parts[0]) / SR + for j, i in enumerate(ids): + u = trim(U[i]); spans.append((t, t + len(u) / SR)); parts.append(u); t += len(u) / SR + g = r.standard_normal(int(r.uniform(.3, 2.0) * SR)) * 1e-3; parts.append(g); t += len(g) / SR + y = np.concatenate(parts).astype(np.float32) + sp = np.concatenate([y[int(s * SR):int(e * SR)] for s, e in spans]); P = (sp ** 2).mean() + for cond, kind, snr in (("clean", None, None), ("white5", "white", 5), ("pink0", "pink", 0)): + z = y + if kind: + rr = np.random.default_rng(3000 + k) + n = rr.standard_normal(len(y)) if kind == "white" else pink(len(y), rr) + z = np.clip(y + n * np.sqrt(P / 10 ** (snr / 10)), -1, 1).astype(np.float32) + nm = f"sn_{cond}_{k:02d}"; sf.write(f"{D}/{nm}.wav", z, SR, subtype="PCM_16") + man.append(dict(name=nm, kind="sinr", cond=cond, ref=" ".join(T[i] for i in ids))) +json.dump(man, open(f"{D}/manifest.json", "w")) +from collections import Counter +print(len(man), Counter(m["kind"] for m in man)) diff --git a/scripts/vad_bench/decoder_guards/results/tables.txt b/scripts/vad_bench/decoder_guards/results/tables.txt new file mode 100644 index 0000000..393985e --- /dev/null +++ b/scripts/vad_bench/decoder_guards/results/tables.txt @@ -0,0 +1,70 @@ +uptime when the tables were made: ['21.18', '43.26', '43.99'] (load average 1, 5, 15 min) + +== byte check: --vad-trim 0 against the old binary (JSON output, every file run both ways) +Ultra head 15/15 identical +Redux head 15/15 identical +v3 + Silero 15/15 identical + +== word error rate (%), words in the reference in brackets +-- talks: 3 files, 5627 words + old trim 0.3 +filter 0.5 trim - old filter - trim +Ultra head 3.68 3.68 3.68 +0.00 +0.00 +Redux head 4.39 4.51 - +0.12 - +v3 + Silero 3.45 3.45 3.45 +0.00 +0.00 +-- speech in noise: clean: 4 files, 429 words + old trim 0.3 +filter 0.5 trim - old filter - trim +Ultra head 2.10 1.86 1.86 -0.23 +0.00 +Redux head 3.26 3.03 - -0.23 - +v3 + Silero 1.86 1.86 1.86 +0.00 +0.00 +-- speech in noise: white5: 4 files, 429 words + old trim 0.3 +filter 0.5 trim - old filter - trim +Ultra head 4.90 4.43 4.43 -0.47 +0.00 +Redux head 7.23 6.76 - -0.47 - +v3 + Silero 5.59 5.13 5.13 -0.47 +0.00 +-- speech in noise: pink0: 4 files, 429 words + old trim 0.3 +filter 0.5 trim - old filter - trim +Ultra head 6.53 5.59 5.59 -0.93 +0.00 +Redux head 12.35 13.29 - +0.93 - +v3 + Silero 7.93 7.69 7.69 -0.23 +0.00 +-- talk DanielKahneman-merged +Ultra head 3.12 3.18 3.18 +Redux head 4.18 4.08 - +v3 + Silero 2.55 2.55 2.55 +-- talk EricMead_2009P_EricMead-merged +Ultra head 5.17 5.10 5.10 +Redux head 5.56 5.96 - +v3 + Silero 5.43 5.43 5.43 +-- talk GaryFlake-merged +Ultra head 3.18 3.09 3.09 +Redux head 3.36 3.72 - +v3 + Silero 3.18 3.18 3.18 + +== noise block of 60 s inside speech (insert files): seconds of the block that the decoder gets, and words in it +files: 30 (3 talks x white, pink, clicks, music at -20 and -5 dB, white and music at -35 dB) + sec old sec new words old words new +filter 0.5 files with words old/new/f05 +Ultra head 33.8 12.0 0 0 0 0/0/0 +Redux head 29.7 0.3 0 0 - 0/0/- +v3 + Silero 27.7 0.1 11 0 0 5/0/0 +per noise type, seconds of the block decoded (old -> new): +Ultra head clicks-20 26->0 clicks-5 48->7 music-20 13->0 music-35 24->0 music-5 13->0 pink-20 27->0 pink-5 60->52 white-20 47->1 white-35 21->0 white-5 60->58 +Redux head clicks-20 25->0 clicks-5 25->0 music-20 18->0 music-35 25->0 music-5 17->0 pink-20 37->0 pink-5 60->1 white-20 25->0 white-35 25->0 white-5 39->0 +v3 + Silero clicks-20 28->0 clicks-5 28->0 music-20 28->0 music-35 28->0 music-5 28->0 pink-20 28->0 pink-5 28->0 white-20 28->0 white-35 28->0 white-5 28->0 + +== what the filter (0.5) took out of the files with real speech +Ultra head words decoded 15375, hallucinated words in the noise blocks removed 0, other words removed 0 +v3 + Silero words decoded 15406, hallucinated words in the noise blocks removed 0, other words removed 3 + +== 30 s of noise alone, decoded whole (no VAD): every word is an invented word +files: 63 (3 talks x white, pink, clicks, music, hum, tone, sweep at -35, -20, -5 dB against the speech level) + files with words words +filter 0.5: files words +Ultra head 0 0 0 0 +Redux head 0 0 - - +v3 + Silero 1 1 0 0 + +== the filter alone (old cuts, --vad-trim 0, --min-local-conf 0.5) on the insert files whose noise block gave words +Ultra head 0 files: words in the block 0 -> 0; other words of those files 0, lost 0 +v3 + Silero 5 files: words in the block 11 -> 1; other words of those files 1484, lost 0 + +== higher thresholds on the speech in noise files (WER %, trim 0.3; words dropped of the words decoded) +Ultra head 12 files, 1287 reference words off: 3.96 (0 dropped) 0.5: 3.96 (0 dropped) 0.7: 3.96 (0 dropped) 0.9: 6.06 (31 dropped) +v3 + Silero 12 files, 1287 reference words off: 4.90 (0 dropped) 0.5: 4.90 (0 dropped) 0.7: 4.90 (0 dropped) 0.9: 16.08 (164 dropped) diff --git a/scripts/vad_bench/decoder_guards/run_all.py b/scripts/vad_bench/decoder_guards/run_all.py new file mode 100644 index 0000000..dd06289 --- /dev/null +++ b/scripts/vad_bench/decoder_guards/run_all.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""usage: run_all.py DATA OUT --old CLI_OLD --new CLI_NEW --ultra GGUF --redux GGUF --v3 GGUF --silero GGUF [-j N] + +Runs the CLI over the corpus of make_corpus.py and keeps every JSON output in OUT. CLI_OLD is +parakeet-cli built from the commit before the trim change, CLI_NEW the one with it. Per detector +(ultra head, redux head, v3 with Silero): + .old/.json transcribe --vad --json with CLI_OLD + .new/.json the same with CLI_NEW (trim 0.3, the default) + .new0/.json CLI_NEW with --vad-trim 0 (must equal .old byte for byte; a subset) + .f05/.json CLI_NEW with --min-local-conf 0.5 (ultra and v3 only) + .plain/.json, .plainf05/.json noise-only files, no VAD, without and with the filter + .old0f05/.json CLI_NEW with --vad-trim 0 --min-local-conf 0.5, only on the insert files in whose noise + block the old cuts gave words (the filter on its own, without the trim) + .f07, .f09/.json thresholds 0.7 and 0.9 on the speech in noise files (ultra and v3) + .segold/.json, .segnew/.json `vad --mode segments` of insert files (cuts) +Existing outputs are kept, so a run can be resumed. Not a timing run.""" +import argparse, json, os, subprocess, sys +from concurrent.futures import ThreadPoolExecutor + +ap = argparse.ArgumentParser() +ap.add_argument("data"); ap.add_argument("out") +for k in ("old", "new", "ultra", "redux", "v3", "silero"): ap.add_argument("--" + k, required=True) +ap.add_argument("--only", default="", help="regular expression: run only the files whose name matches") +ap.add_argument("-j", type=int, default=3); ap.add_argument("--sets", type=int, default=4, help="LibriSpeech sets per condition") +a = ap.parse_args() +man = [m for m in json.load(open(f"{a.data}/manifest.json")) + if m["kind"] != "sinr" or int(m["name"].rsplit("_", 1)[1]) < a.sets] +import re +if a.only: man = [m for m in man if re.search(a.only, m["name"])] +DET = {"ultra": (a.ultra, None), "redux": (a.redux, None), "v3": (a.v3, a.silero)} +jobs = [] +def add(out, cmd): + if not os.path.exists(out) or os.path.getsize(out) == 0: jobs.append((out, cmd)) +for det, (model, sil) in DET.items(): + vad = ["--vad"] + (["--vad-model", sil] if sil else []) + for mf in man: + wav = f"{a.data}/{mf['name']}.wav" + if mf["kind"] == "noise": # no VAD: the whole file is decoded, every word is an invented word + plain = ["transcribe", "--model", model, "--input", wav, "--json"] + for tag, extra in (("plain", []), ("plainf05", ["--min-local-conf", "0.5"])): + if tag == "plainf05" and det == "redux": continue + os.makedirs(f"{a.out}/{det}.{tag}", exist_ok=True) + add(f"{a.out}/{det}.{tag}/{mf['name']}.json", [a.new] + plain + extra) + continue + base = ["transcribe", "--model", model, "--input", wav, "--json"] + vad + def o(tag): os.makedirs(f"{a.out}/{det}.{tag}", exist_ok=True); return f"{a.out}/{det}.{tag}/{mf['name']}.json" + add(o("old"), [a.old] + base) + add(o("new"), [a.new] + base) + if mf["kind"] == "talk" or mf["name"].endswith(("_00", "_01")) or mf["name"].endswith(("white-20", "music-35")): + add(o("new0"), [a.new] + base + ["--vad-trim", "0"]) + if det != "redux": + add(o("f05"), [a.new] + base + ["--min-local-conf", "0.5"]) + if mf["kind"] == "insert": + seg = ["vad", "--model", sil or model, "--input", wav, "--mode", "segments"] + add(o("segold"), [a.old] + seg) + add(o("segnew"), [a.new] + seg) +print(len(jobs), "jobs", flush=True) +def run(j): + out, cmd = j + r = subprocess.run(cmd, capture_output=True, text=True) + if r.returncode == 0: open(out, "w").write(r.stdout) + else: print("FAILED", out, r.stderr[-300:], flush=True) + return out +with ThreadPoolExecutor(a.j) as ex: + for i, o in enumerate(ex.map(run, jobs)): + if i % 20 == 0: print(i, "/", len(jobs), flush=True) + +# Second pass: the filter alone, on the files where the old cuts gave invented words in the noise block. +jobs = [] +for det, (model, sil) in DET.items(): + vad = ["--vad"] + (["--vad-model", sil] if sil else []) + if det == "redux": continue + for mf in man: + if mf["kind"] != "insert": continue + f = f"{a.out}/{det}.old/{mf['name']}.json" + if not os.path.exists(f) or os.path.getsize(f) == 0: continue + lo, hi = mf["ins"] + if not any(lo + 0.5 <= w["start"] <= hi - 0.5 for w in json.load(open(f))["words"]): continue + os.makedirs(f"{a.out}/{det}.old0f05", exist_ok=True) + add(f"{a.out}/{det}.old0f05/{mf['name']}.json", + [a.new, "transcribe", "--model", model, "--input", f"{a.data}/{mf['name']}.wav", "--json"] + vad + ["--vad-trim", "0", "--min-local-conf", "0.5"]) +for det, (model, sil) in DET.items(): + if det == "redux": continue + vad = ["--vad"] + (["--vad-model", sil] if sil else []) + for mf in man: + if mf["kind"] != "sinr": continue + for tag, thr in (("f07", "0.7"), ("f09", "0.9")): + os.makedirs(f"{a.out}/{det}.{tag}", exist_ok=True) + add(f"{a.out}/{det}.{tag}/{mf['name']}.json", + [a.new, "transcribe", "--model", model, "--input", f"{a.data}/{mf['name']}.wav", "--json"] + vad + ["--min-local-conf", thr]) +print(len(jobs), "filter-only jobs", flush=True) +with ThreadPoolExecutor(a.j) as ex: + list(ex.map(run, jobs)) diff --git a/scripts/vad_bench/decoder_guards/tables.py b/scripts/vad_bench/decoder_guards/tables.py new file mode 100644 index 0000000..cb5928b --- /dev/null +++ b/scripts/vad_bench/decoder_guards/tables.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""usage: tables.py DATA OUT > results/tables.txt + +Reads the outputs of run_all.py and prints the tables of docs/vad-benchmarks.md (trim and word filter).""" +import json, os, re, sys, glob +import numpy as np + +D, O = sys.argv[1], sys.argv[2] +man = {m["name"]: m for m in json.load(open(f"{D}/manifest.json"))} +DETS = [("ultra", "Ultra head"), ("redux", "Redux head"), ("v3", "v3 + Silero")] + +def norm(s): return re.sub(r"[^a-z0-9' ]", " ", s.lower().replace("-", " ")).split() +def ed(a, b): + prev = list(range(len(b) + 1)) + for i, x in enumerate(a, 1): + cur = [i] + [0] * len(b) + for j, y in enumerate(b, 1): cur[j] = min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (x != y)) + prev = cur + return prev[-1] +def load(det, tag, name): + f = f"{O}/{det}.{tag}/{name}.json" + if not os.path.exists(f) or os.path.getsize(f) == 0: return None + return json.load(open(f)) +def wer(det, tag, names): + e = n = 0 + for nm in names: + j = load(det, tag, nm) + if j is None: return None + ref = norm(man[nm]["ref"]); e += ed(norm(j["text"]), ref); n += len(ref) + return 100.0 * e / n, n +def fmt(x): return "-" if x is None else f"{x[0]:.2f}" +def overlap(segs, a, b): return sum(max(0.0, min(s["end"], b) - max(s["start"], a)) for s in segs) + +print("uptime when the tables were made:", open("/proc/loadavg").read().split()[:3], "(load average 1, 5, 15 min)") +print("\n== byte check: --vad-trim 0 against the old binary (JSON output, every file run both ways)") +for det, lab in DETS: + eq = tot = 0 + for f in glob.glob(f"{O}/{det}.new0/*.json"): + nm = os.path.basename(f)[:-5]; o = f"{O}/{det}.old/{nm}.json" + if os.path.exists(o): tot += 1; eq += open(f).read() == open(o).read() + print(f"{lab:12s} {eq}/{tot} identical") + +print("\n== word error rate (%), words in the reference in brackets") +groups = [("talks", [n for n in man if man[n]["kind"] == "talk"])] +for c in ("clean", "white5", "pink0"): + groups.append((f"speech in noise: {c}", [n for n in man if man[n]["kind"] == "sinr" and man[n]["cond"] == c and load("ultra", "old", n) is not None])) +for gname, names in groups: + nref = sum(len(norm(man[n]["ref"])) for n in names) + print(f"-- {gname}: {len(names)} files, {nref} words") + print(f"{'':14s}{'old':>8s}{'trim 0.3':>10s}{'+filter 0.5':>13s}{'trim - old':>12s}{'filter - trim':>15s}") + for det, lab in DETS: + o, t, f = wer(det, "old", names), wer(det, "new", names), wer(det, "f05", names) + d1 = "-" if not (o and t) else f"{t[0] - o[0]:+.2f}" + d2 = "-" if not (t and f) else f"{f[0] - t[0]:+.2f}" + print(f"{lab:14s}{fmt(o):>8s}{fmt(t):>10s}{fmt(f):>13s}{d1:>12s}{d2:>15s}") +for tk in sorted({m["talk"] for m in man.values() if m["kind"] == "talk"}): + print(f"-- talk {tk}") + for det, lab in DETS: + o, t, f = wer(det, "old", [f"talk_{tk}"]), wer(det, "new", [f"talk_{tk}"]), wer(det, "f05", [f"talk_{tk}"]) + print(f"{lab:14s}{fmt(o):>8s}{fmt(t):>10s}{fmt(f):>13s}") + +print("\n== noise block of 60 s inside speech (insert files): seconds of the block that the decoder gets, and words in it") +ins = [n for n in man if man[n]["kind"] == "insert"] +print(f"files: {len(ins)} (3 talks x white, pink, clicks, music at -20 and -5 dB, white and music at -35 dB)") +print(f"{'':14s}{'sec old':>9s}{'sec new':>9s}{'words old':>11s}{'words new':>11s}{'+filter 0.5':>13s}{'files with words old/new/f05':>32s}") +def inblock(j, a, b): + return [w for w in j["words"] if a + 0.5 <= w["start"] <= b - 0.5] +for det, lab in DETS: + so, sn, wo, wn, wf, fo, fn, ff = [], [], 0, 0, 0, 0, 0, 0 + for nm in ins: + a, b = man[nm]["ins"] + jo, jn, jf = load(det, "old", nm), load(det, "new", nm), load(det, "f05", nm) + so_, sn_ = load(det, "segold", nm), load(det, "segnew", nm) + if so_ and sn_: so.append(overlap(so_["segments"], a, b)); sn.append(overlap(sn_["segments"], a, b)) + if jo: k = len(inblock(jo, a, b)); wo += k; fo += k > 0 + if jn: k = len(inblock(jn, a, b)); wn += k; fn += k > 0 + if jf: k = len(inblock(jf, a, b)); wf += k; ff += k > 0 + print(f"{lab:14s}{np.mean(so):9.1f}{np.mean(sn):9.1f}{wo:11d}{wn:11d}{(str(wf) if det != 'redux' else '-'):>13s}{f'{fo}/{fn}/{ff}' if det != 'redux' else f'{fo}/{fn}/-':>32s}") +print("per noise type, seconds of the block decoded (old -> new):") +for det, lab in DETS: + row = {} + for nm in ins: + a, b = man[nm]["ins"]; so_, sn_ = load(det, "segold", nm), load(det, "segnew", nm) + if so_ and sn_: row.setdefault(f"{man[nm]['noise']}{man[nm]['level']}", []).append((overlap(so_["segments"], a, b), overlap(sn_["segments"], a, b))) + print(f"{lab:14s}" + " ".join(f"{k} {np.mean([x[0] for x in v]):.0f}->{np.mean([x[1] for x in v]):.0f}" for k, v in sorted(row.items()))) + +print("\n== what the filter (0.5) took out of the files with real speech") +for det, lab in DETS: + if det == "redux": continue + dropped = 0; words = 0; hall = 0 + for nm, m in man.items(): + j, t = load(det, "f05", nm), load(det, "new", nm) + if j is None or t is None: continue + d = j.get("guard", {}).get("dropped_words", 0) + if m["kind"] == "insert": + a, b = m["ins"]; hall += len(inblock(t, a, b)) - len(inblock(j, a, b)) + # words outside the block that went with it + dropped += d - (len(inblock(t, a, b)) - len(inblock(j, a, b))) + else: dropped += d + words += len(t["words"]) + print(f"{lab:12s} words decoded {words}, hallucinated words in the noise blocks removed {hall}, other words removed {dropped}") + +print("\n== 30 s of noise alone, decoded whole (no VAD): every word is an invented word") +nz = [n for n in man if man[n]["kind"] == "noise"] +print(f"files: {len(nz)} (3 talks x white, pink, clicks, music, hum, tone, sweep at -35, -20, -5 dB against the speech level)") +print(f"{'':14s}{'files with words':>18s}{'words':>8s}{'+filter 0.5: files':>20s}{'words':>8s}") +for det, lab in DETS: + fw = w = ffw = fw5 = 0 + for nm in nz: + j, f = load(det, "plain", nm), load(det, "plainf05", nm) + if j is None: continue + k = len(j["words"]); w += k; fw += k > 0 + if f is not None: fw5 += len(f["words"]); ffw += len(f["words"]) > 0 + print(f"{lab:14s}{fw:18d}{w:8d}{(str(ffw) if det != 'redux' else '-'):>20s}{(str(fw5) if det != 'redux' else '-'):>8s}") + +print("\n== the filter alone (old cuts, --vad-trim 0, --min-local-conf 0.5) on the insert files whose noise block gave words") +for det, lab in DETS: + if det == "redux": continue + files = hall = left = lost = real = 0 + for nm in ins: + j, f = load(det, "old", nm), load(det, "old0f05", nm) + if j is None or f is None: continue + a, b = man[nm]["ins"]; files += 1 + k0, k1 = len(inblock(j, a, b)), len(inblock(f, a, b)) + hall += k0; left += k1 + lost += len(j["words"]) - len(f["words"]) - (k0 - k1); real += len(j["words"]) - k0 + print(f"{lab:12s} {files} files: words in the block {hall} -> {left}; other words of those files {real}, lost {lost}") + +print("\n== higher thresholds on the speech in noise files (WER %, trim 0.3; words dropped of the words decoded)") +sn = [n for n in man if man[n]["kind"] == "sinr" and load("ultra", "f05", n) is not None] +for det, lab in DETS: + if det == "redux": continue + row = [] + for tag, t in (("new", "off"), ("f05", "0.5"), ("f07", "0.7"), ("f09", "0.9")): + w = wer(det, tag, [n for n in sn if load(det, tag, n) is not None]) + dr = sum(load(det, tag, n).get("guard", {}).get("dropped_words", 0) for n in sn if load(det, tag, n) is not None) + row.append(f"{t}: {fmt(w)} ({dr} dropped)") + print(f"{lab:12s} {len(sn)} files, {sum(len(norm(man[n]['ref'])) for n in sn)} reference words " + " ".join(row)) diff --git a/src/model.cpp b/src/model.cpp index 4c0bb42..d1d25a5 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -462,7 +462,10 @@ std::vector vad_slices(const Model& m, const std::vector& pcm16k, std::string Model::transcribe_pcm_vad(const std::vector& pcm, int sample_rate, Decoder decoder, const std::string& target_lang, const SegmenterOpts& opts, - const VadProbabilityFn* external_vad) const { + const VadProbabilityFn* external_vad, + const WordFilter& filter) const { + if (filter.active()) + return transcribe_pcm_vad_with_timestamps(pcm, sample_rate, decoder, target_lang, opts, external_vad, filter).text; PoolLease lease(pool_snapshot()); if (!external_vad && !loader_.config().vad.present) throw std::runtime_error("model has no VAD head"); const std::vector pcm16k = @@ -486,22 +489,28 @@ std::string Model::transcribe_pcm_vad(const std::vector& pcm, int sample_ Transcription Model::transcribe_pcm_vad_with_timestamps(const std::vector& pcm, int sample_rate, Decoder decoder, const std::string& target_lang, const SegmenterOpts& opts, - const VadProbabilityFn* external_vad) const { + const VadProbabilityFn* external_vad, + const WordFilter& filter) const { PoolLease lease(pool_snapshot()); if (!external_vad && !loader_.config().vad.present) throw std::runtime_error("model has no VAD head"); const std::vector pcm16k = sample_rate == 16000 ? pcm : resample_linear(pcm, sample_rate, 16000); - if ((double)pcm16k.size() / 16000.0 <= opts.max_seg_sec) - return transcribe_with_timestamps(pcm16k, 16000, decoder, target_lang); + if ((double)pcm16k.size() / 16000.0 <= opts.max_seg_sec) { + Transcription t = transcribe_with_timestamps(pcm16k, 16000, decoder, target_lang); + apply_word_filter(t, filter); + return t; + } const std::vector slices = vad_slices(*this, pcm16k, opts, external_vad); - if (slices.empty()) return Transcription(); // no speech found Transcription all; + if (filter.active()) all.dropped_words = 0; + if (slices.empty()) return all; // no speech found std::vector*> pcms; for (const Slice& s : slices) pcms.push_back(&s.pcm); std::vector parts = transcribe_16k_grouped(pcms, decoder, target_lang, true); for (size_t i = 0; i < slices.size(); ++i) { const Slice& s = slices[i]; Transcription& t = parts[i]; + if (filter.active()) all.dropped_words += apply_word_filter(t, filter); for (Word& w : t.words) { w.start += (float)s.start_sec; w.end += (float)s.start_sec; } for (TokenInfo& k : t.tokens) k.frame += s.start_frame; if (!t.text.empty()) { diff --git a/src/model.hpp b/src/model.hpp index 9945842..6a5317b 100644 --- a/src/model.hpp +++ b/src/model.hpp @@ -162,17 +162,25 @@ class Model { // PCM to one speech probability per opts.frame_sec seconds, and opts.frame_sec // must then be set to that period. The segmenter, the 30 s cap and the decode // of each segment are the same as with the head. + // + // `filter` (optional, off by default, see WordFilter) removes words by their + // local confidence. It runs on each decode unit alone: the whole clip when it + // is at most opts.max_seg_sec, else each VAD segment. With a filter on, the + // text path also runs the timestamp decode, and the Transcription reports + // the words dropped in `dropped_words`. using VadProbabilityFn = std::function(const std::vector&)>; std::string transcribe_pcm_vad(const std::vector& pcm, int sample_rate, Decoder decoder = Decoder::kDefault, const std::string& target_lang = "", const SegmenterOpts& opts = SegmenterOpts(), - const VadProbabilityFn* external_vad = nullptr) const; + const VadProbabilityFn* external_vad = nullptr, + const WordFilter& filter = WordFilter()) const; Transcription transcribe_pcm_vad_with_timestamps( const std::vector& pcm, int sample_rate, Decoder decoder = Decoder::kDefault, const std::string& target_lang = "", const SegmenterOpts& opts = SegmenterOpts(), - const VadProbabilityFn* external_vad = nullptr) const; + const VadProbabilityFn* external_vad = nullptr, + const WordFilter& filter = WordFilter()) const; // The underlying loaded GGUF. Exposed so the streaming C-API can build a // pk::StreamingSession (and a MelFrontend) over the same load-once model. diff --git a/src/parakeet_capi.cpp b/src/parakeet_capi.cpp index b3d5e5c..0d73fc9 100644 --- a/src/parakeet_capi.cpp +++ b/src/parakeet_capi.cpp @@ -651,6 +651,43 @@ extern "C" char* parakeet_capi_transcribe_path_json(parakeet_ctx* ctx, } } +extern "C" char* parakeet_capi_transcribe_path_json_with(parakeet_ctx* ctx, const char* wav_path, + int decoder, const char* options_json) { + if (!ctx) return nullptr; + if (!ctx->model) { + ctx->last_error = ctx->diar + ? "context holds a diarization model; use parakeet_capi_diarize_*" + : no_model_msg(ctx); + return nullptr; + } + if (!wav_path) { ctx->last_error = "wav_path is NULL"; return nullptr; } + try { + pk::WordFilter filter; + std::string err; + if (!pk::parse_filter_options(options_json, filter, err)) { + ctx->last_error = err; + return nullptr; + } + pk::Transcription tr = + ctx->model->transcribe_path_with_timestamps(wav_path, to_decoder(decoder)); + pk::apply_word_filter(tr, filter); + const pk::ParakeetConfig& cfg = ctx->model->config(); + const float frame_sec = + (float)cfg.hop_length * (float)cfg.subsampling_factor / (float)cfg.sample_rate; + std::string json = pk::transcription_to_json(tr, frame_sec); + ctx->last_error.clear(); + char* out = dup_to_c(json); + if (!out) { ctx->last_error = "out of memory"; return nullptr; } + return out; + } catch (const std::exception& e) { + ctx->last_error = e.what(); + return nullptr; + } catch (...) { + ctx->last_error = "unknown error"; + return nullptr; + } +} + extern "C" char* parakeet_capi_transcribe_path_json_vad(parakeet_ctx* ctx, const char* wav_path, int decoder) { @@ -773,7 +810,7 @@ extern "C" char* parakeet_capi_transcribe_path_json_vad_with(parakeet_ctx* ctx, pk::VadRequest req; std::string err; if (!pk::parse_vad_options(options_json, req, err, - vad_ctx ? pk::VadKind::kSilero : pk::VadKind::kHead)) { + vad_ctx ? pk::VadKind::kSilero : pk::VadKind::kHead, /*allow_filter=*/true)) { ctx->last_error = err; return nullptr; } @@ -793,7 +830,7 @@ extern "C" char* parakeet_capi_transcribe_path_json_vad_with(parakeet_ctx* ctx, } pk::SegmenterOpts so = req.opts; pk::Transcription tr = ctx->model->transcribe_pcm_vad_with_timestamps( - audio.samples, audio.sample_rate, to_decoder(decoder), "", so, vad_ctx ? &fn : nullptr); + audio.samples, audio.sample_rate, to_decoder(decoder), "", so, vad_ctx ? &fn : nullptr, req.filter); const pk::ParakeetConfig& cfg = ctx->model->config(); const float frame_sec = (float)cfg.hop_length * (float)cfg.subsampling_factor / (float)cfg.sample_rate; diff --git a/src/transcription.cpp b/src/transcription.cpp index c8623b5..fe33c53 100644 --- a/src/transcription.cpp +++ b/src/transcription.cpp @@ -3,6 +3,8 @@ #include #include +#include +#include #include #include #include @@ -184,6 +186,7 @@ std::vector group_words(const std::vector& tokens, w.start = (float)start[prev] * frame_sec; w.end = (float)end[built.back()] * frame_sec; w.conf = min_conf(built); + w.tok_first = built.front(); w.tok_last = built.back(); words.push_back(std::move(w)); } built.clear(); @@ -199,6 +202,7 @@ std::vector group_words(const std::vector& tokens, if (!lw.text.empty() && lw.text.back() == ' ') lw.text.pop_back(); lw.text += ct; lw.conf = std::min(lw.conf, conf[i]); + lw.tok_last = i; } else if (curr_punct && !built.empty()) { // Punctuation closing an open word: drop a trailing delimiter token, // then append this token. @@ -224,6 +228,7 @@ std::vector group_words(const std::vector& tokens, w.start = (float)start[prev] * frame_sec; w.end = (float)end[built.back()] * frame_sec; w.conf = min_conf(built); + w.tok_first = built.front(); w.tok_last = built.back(); words.push_back(std::move(w)); } } else if (!built.empty()) { @@ -232,10 +237,112 @@ std::vector group_words(const std::vector& tokens, w.start = (float)start[0] * frame_sec; w.end = (float)end[built.back()] * frame_sec; w.conf = min_conf(built); + w.tok_first = built.front(); w.tok_last = built.back(); words.push_back(std::move(w)); } return words; } +namespace { + +// True when the word has no letter or digit: only punctuation or spaces. +// ASCII is classified directly. Other code points count as content except the +// common punctuation blocks (General Punctuation, CJK symbols and punctuation, +// and the Latin-1 marks used in Spanish and French). +bool is_punct_only(const std::string& w) { + size_t i = 0; + while (i < w.size()) { + const unsigned char c = (unsigned char)w[i]; + uint32_t cp = c; + size_t len = 1; + if (c >= 0xF0 && i + 3 < w.size()) { cp = ((c & 0x07u) << 18) | (((unsigned char)w[i + 1] & 0x3Fu) << 12) | (((unsigned char)w[i + 2] & 0x3Fu) << 6) | ((unsigned char)w[i + 3] & 0x3Fu); len = 4; } + else if (c >= 0xE0 && i + 2 < w.size()) { cp = ((c & 0x0Fu) << 12) | (((unsigned char)w[i + 1] & 0x3Fu) << 6) | ((unsigned char)w[i + 2] & 0x3Fu); len = 3; } + else if (c >= 0xC0 && i + 1 < w.size()) { cp = ((c & 0x1Fu) << 6) | ((unsigned char)w[i + 1] & 0x3Fu); len = 2; } + i += len; + if (cp < 0x80) { + if (std::isalnum((int)cp)) return false; + continue; + } + const bool punct = (cp >= 0x2000 && cp <= 0x206F) || (cp >= 0x3000 && cp <= 0x303F) || + cp == 0xA1 || cp == 0xA7 || cp == 0xAB || cp == 0xB6 || cp == 0xB7 || + cp == 0xBB || cp == 0xBF; + if (!punct) return false; + } + return true; +} + +} // namespace + +int apply_word_filter(Transcription& t, const WordFilter& f) { + if (!f.active() || t.words.empty()) return 0; + const size_t n = t.words.size(); + std::vector drop(n, 0); + size_t dropped = 0; + if (f.min_local_conf > 0.0f) { + // Mean confidence of the words that start within the radius of a word, + // the word itself included. Words are in time order, so the window of + // each word is a range that moves forward; fall back to a scan when a + // caller passes words out of order. + bool sorted = true; + for (size_t i = 1; i < n && sorted; ++i) sorted = t.words[i - 1].start <= t.words[i].start; + std::vector cum(n + 1, 0.0); + for (size_t i = 0; i < n; ++i) cum[i + 1] = cum[i] + (double)t.words[i].conf; + const float r = f.local_radius_sec; + size_t lo = 0, hi = 0; // window [lo, hi) + for (size_t i = 0; i < n; ++i) { + const float s = t.words[i].start; + double sum = 0.0; + size_t cnt = 0; + if (sorted) { + while (lo < i && s - t.words[lo].start > r) ++lo; + if (hi < i + 1) hi = i + 1; + while (hi < n && t.words[hi].start - s <= r) ++hi; + sum = cum[hi] - cum[lo]; + cnt = hi - lo; + } else { + for (size_t j = 0; j < n; ++j) + if (std::fabs(t.words[j].start - s) <= r) { sum += (double)t.words[j].conf; ++cnt; } + } + if (cnt > 0 && sum / (double)cnt < (double)f.min_local_conf) drop[i] = 1; + } + } + if (f.drop_punct_only) + for (size_t i = 0; i < n; ++i) + if (is_punct_only(t.words[i].text)) drop[i] = 1; + for (size_t i = 0; i < n; ++i) dropped += drop[i] ? 1u : 0u; + t.dropped_words = (t.dropped_words < 0 ? 0 : t.dropped_words) + (int)dropped; + if (dropped == 0) return 0; + + // Rebuild words, text and tokens without the dropped words. + std::vector tdrop(t.tokens.size(), 0); + std::vector kept; + std::string text; + for (size_t i = 0; i < n; ++i) { + const Word& w = t.words[i]; + if (drop[i]) { + for (int32_t k = w.tok_first; k >= 0 && k <= w.tok_last && (size_t)k < tdrop.size(); ++k) tdrop[(size_t)k] = 1; + continue; + } + if (!text.empty()) text += ' '; + text += w.text; + kept.push_back(w); + } + std::vector remap(t.tokens.size(), -1); + std::vector toks; + for (size_t k = 0; k < t.tokens.size(); ++k) + if (!tdrop[k]) { remap[k] = (int32_t)toks.size(); toks.push_back(t.tokens[k]); } + for (Word& w : kept) { + int32_t a = -1, b = -1; + for (int32_t k = w.tok_first; k >= 0 && k <= w.tok_last && (size_t)k < remap.size(); ++k) + if (remap[(size_t)k] >= 0) { if (a < 0) a = remap[(size_t)k]; b = remap[(size_t)k]; } + w.tok_first = a; + w.tok_last = b; + } + t.words = std::move(kept); + t.tokens = std::move(toks); + t.text = std::move(text); + return (int)dropped; +} + } // namespace pk diff --git a/src/transcription.hpp b/src/transcription.hpp index 59d71ad..faddac4 100644 --- a/src/transcription.hpp +++ b/src/transcription.hpp @@ -19,6 +19,10 @@ struct Word { float start = 0.0f; float end = 0.0f; float conf = 0.0f; + // Indices of the word's first and last token in Transcription::tokens, or + // -1 when unknown. Used to remove a word's tokens with it. + int32_t tok_first = -1; + int32_t tok_last = -1; }; // A full transcription result: the flat text, the per-word timestamps + @@ -27,6 +31,9 @@ struct Transcription { std::string text; std::vector words; std::vector tokens; + // Number of words an optional word filter removed (see WordFilter). -1 when + // no filter ran, so a caller can tell "nothing dropped" from "no filter". + int dropped_words = -1; }; // Group a per-token decode (TokenInfo sequence, in emission order) into words, @@ -56,4 +63,31 @@ std::vector group_words(const std::vector& tokens, const std::vector& pieces, float frame_sec); +// Optional word filter, off by default. It removes words from a Transcription +// by their own confidence values, with no change to the decoding. +// +// min_local_conf : 0 = off. A word is dropped when the mean confidence of the +// words that start within local_radius_sec of its start (the +// word included) is below this value. A low confidence word +// between confident words keeps a high mean and stays. Words +// that stand alone, or only among other low confidence words, +// are dropped. This is what noise tends to produce. +// local_radius_sec: window for the mean, both sides, seconds (default 5). +// drop_punct_only: also drop words that are only punctuation (a CTC model +// can emit a lone "." or "?" on noise). +// +// A "decode unit" is one call of the decoder: the whole clip, or one VAD +// segment. Apply the filter to each unit alone, so that the mean never spans +// two segments. The dropped words leave `text`, `words` and `tokens`, and the +// token indices of the kept words are updated. When nothing is dropped the +// transcription is not changed. Returns the number of words dropped and adds it +// to `dropped_words` (which becomes 0 or more). +struct WordFilter { + float min_local_conf = 0.0f; + float local_radius_sec = 5.0f; + bool drop_punct_only = false; + bool active() const { return min_local_conf > 0.0f || drop_punct_only; } +}; +int apply_word_filter(Transcription& t, const WordFilter& f); + } // namespace pk diff --git a/src/transcription_json.cpp b/src/transcription_json.cpp index 7d81de5..4509259 100644 --- a/src/transcription_json.cpp +++ b/src/transcription_json.cpp @@ -88,7 +88,14 @@ std::string transcription_to_json(const Transcription& tr, float frame_sec) { append_json_float(out, "%.4f", tr.tokens[i].conf); out += '}'; } - out += "]}"; + out += ']'; + // Present only when a word filter ran: how many words it removed. + if (tr.dropped_words >= 0) { + out += ",\"guard\":{\"dropped_words\":"; + append_json_int(out, tr.dropped_words); + out += '}'; + } + out += '}'; return out; } diff --git a/src/transcription_json.hpp b/src/transcription_json.hpp index 1bda990..0413572 100644 --- a/src/transcription_json.hpp +++ b/src/transcription_json.hpp @@ -14,7 +14,8 @@ void append_json_int(std::string& out, int v); void append_json_float(std::string& out, const char* fmt, float v); // Serialize a Transcription to the C-API JSON document shape: -// {"text", "frame_sec", "words", "tokens"}. +// {"text", "frame_sec", "words", "tokens"} and, when a word filter ran, +// "guard":{"dropped_words":N}. std::string transcription_to_json(const Transcription& tr, float frame_sec); // Serialize the offline TDT beam result: diff --git a/src/vad_json.cpp b/src/vad_json.cpp index 9d3ce06..ecf5650 100644 --- a/src/vad_json.cpp +++ b/src/vad_json.cpp @@ -47,64 +47,96 @@ bool positive_seconds(double v) { return std::isfinite(v) && v > 0.0 && v <= 1e6 } // namespace -bool parse_vad_options(const char* json, VadRequest& req, std::string& err, VadKind kind) { - req = VadRequest(); - req.kind = kind; - req.opts = default_segmenter_opts(kind); +namespace { + +// Shared parser of the flat option objects. `vad_keys` accepts the segmenter +// and mode keys, `filter_keys` the word filter keys ("trim" belongs to the +// segmenter). `what` names the document in error messages. +bool parse_options(const char* json, VadRequest& req, std::string& err, bool vad_keys, + bool filter_keys, const char* what) { + const std::string docs = std::string(what) + "s"; // "VAD options" / "options" if (!json) return true; Cursor c{json}; c.ws(); if (*c.p == '\0') return true; - if (*c.p != '{') { err = "invalid VAD options: expected a JSON object"; return false; } + if (*c.p != '{') { err = "invalid " + docs + ": expected a JSON object"; return false; } ++c.p; c.ws(); - if (*c.p == '}') { ++c.p; c.ws(); if (*c.p) { err = "invalid VAD options: trailing text"; return false; } return true; } + if (*c.p == '}') { ++c.p; c.ws(); if (*c.p) { err = "invalid " + docs + ": trailing text"; return false; } return true; } for (;;) { c.ws(); std::string key; - if (!parse_string(c, key)) { err = "invalid VAD options: expected a key"; return false; } + if (!parse_string(c, key)) { err = "invalid " + docs + ": expected a key"; return false; } c.ws(); - if (*c.p != ':') { err = "invalid VAD options: expected ':' after \"" + key + "\""; return false; } + if (*c.p != ':') { err = "invalid " + docs + ": expected ':' after \"" + key + "\""; return false; } ++c.p; c.ws(); - if (key == "mode") { + const std::string opt = std::string(what) + " " + key; + const bool is_seg_num = key == "threshold" || key == "min_pause" || key == "min_speech" || + key == "max_segment" || key == "speech_pad" || key == "trim"; + if (vad_keys && key == "mode") { std::string v; - if (!parse_string(c, v)) { err = "invalid VAD option mode: expected a string"; return false; } + if (!parse_string(c, v)) { err = "invalid " + opt + ": expected a string"; return false; } if (v == "speech") req.mode = VadRequest::Mode::kSpeech; else if (v == "segments") req.mode = VadRequest::Mode::kSegments; - else { err = "invalid VAD option mode: use \"speech\" or \"segments\""; return false; } - } else if (key == "probabilities") { - if (std::strncmp(c.p, "true", 4) == 0) { req.probabilities = true; c.p += 4; } - else if (std::strncmp(c.p, "false", 5) == 0) { req.probabilities = false; c.p += 5; } - else { err = "invalid VAD option probabilities: expected true or false"; return false; } - } else if (key == "threshold" || key == "min_pause" || key == "min_speech" || - key == "max_segment" || key == "speech_pad") { + else { err = "invalid " + opt + ": use \"speech\" or \"segments\""; return false; } + } else if ((vad_keys && key == "probabilities") || (filter_keys && key == "drop_punct_only")) { + bool v = false; + if (std::strncmp(c.p, "true", 4) == 0) { v = true; c.p += 4; } + else if (std::strncmp(c.p, "false", 5) == 0) { v = false; c.p += 5; } + else { err = "invalid " + opt + ": expected true or false"; return false; } + if (key == "probabilities") req.probabilities = v; + else req.filter.drop_punct_only = v; + } else if ((vad_keys && is_seg_num) || (filter_keys && (key == "min_local_conf" || key == "local_radius"))) { double v = 0.0; - if (!parse_number(c, v)) { err = "invalid VAD option " + key + ": expected a number"; return false; } + if (!parse_number(c, v)) { err = "invalid " + opt + ": expected a number"; return false; } if (key == "threshold") { - if (!(v > 0.0 && v <= 1.0)) { err = "invalid VAD option threshold: must be in (0, 1]"; return false; } + if (!(v > 0.0 && v <= 1.0)) { err = "invalid " + opt + ": must be in (0, 1]"; return false; } req.opts.threshold = (float)v; - } else if (key == "speech_pad") { - if (!(std::isfinite(v) && v >= 0.0 && v <= 1e6)) { err = "invalid VAD option speech_pad: must be a number of seconds >= 0"; return false; } - req.opts.pad_sec = v; + } else if (key == "min_local_conf") { + if (!(v >= 0.0 && v <= 1.0)) { err = "invalid " + opt + ": must be in [0, 1] (0 = off)"; return false; } + req.filter.min_local_conf = (float)v; + } else if (key == "local_radius") { + if (!positive_seconds(v)) { err = "invalid " + opt + ": must be a number of seconds > 0"; return false; } + req.filter.local_radius_sec = (float)v; + } else if (key == "speech_pad" || key == "trim") { + if (!(std::isfinite(v) && v >= 0.0 && v <= 1e6)) { err = "invalid " + opt + ": must be a number of seconds >= 0"; return false; } + (key == "trim" ? req.opts.trim_sec : req.opts.pad_sec) = v; } else { - if (!positive_seconds(v)) { err = "invalid VAD option " + key + ": must be a number of seconds > 0"; return false; } + if (!positive_seconds(v)) { err = "invalid " + opt + ": must be a number of seconds > 0"; return false; } if (key == "min_pause") req.opts.min_pause_sec = v; else if (key == "min_speech") req.opts.min_speech_sec = v; else req.opts.max_seg_sec = v; } } else { - err = "unknown VAD option: " + key; + err = std::string("unknown ") + what + ": " + key; return false; } c.ws(); if (*c.p == ',') { ++c.p; continue; } if (*c.p == '}') { ++c.p; break; } - err = "invalid VAD options: expected ',' or '}'"; + err = "invalid " + docs + ": expected ',' or '}'"; return false; } c.ws(); - if (*c.p) { err = "invalid VAD options: trailing text"; return false; } + if (*c.p) { err = "invalid " + docs + ": trailing text"; return false; } + return true; +} + +} // namespace + +bool parse_vad_options(const char* json, VadRequest& req, std::string& err, VadKind kind, + bool allow_filter) { + req = VadRequest(); + req.kind = kind; + req.opts = default_segmenter_opts(kind); + return parse_options(json, req, err, true, allow_filter, "VAD option"); +} + +bool parse_filter_options(const char* json, WordFilter& filter, std::string& err) { + VadRequest req; + if (!parse_options(json, req, err, false, true, "option")) return false; + filter = req.filter; return true; } diff --git a/src/vad_json.hpp b/src/vad_json.hpp index db38a0d..5d62423 100644 --- a/src/vad_json.hpp +++ b/src/vad_json.hpp @@ -5,6 +5,7 @@ #include #include +#include "transcription.hpp" #include "vad_segmenter.hpp" namespace pk { @@ -18,16 +19,26 @@ struct VadRequest { Mode mode = Mode::kSpeech; bool probabilities = false; // add "probabilities" to the document VadKind kind = VadKind::kHead; // which model the defaults in `opts` are for + WordFilter filter; // word filter keys, only with allow_filter }; // Parses an options document: a flat JSON object, or NULL / "" for the // defaults. Keys: "threshold" (0 < x <= 1), "min_pause", "min_speech", -// "max_segment" (seconds, > 0), "mode" ("speech" or "segments"), -// "probabilities" (bool), "speech_pad" (seconds >= 0, "speech" mode). Unknown -// keys and bad values are errors. The values that a key leaves out come from -// default_segmenter_opts(kind). Returns false and sets `err` on failure. +// "max_segment" (seconds, > 0), "trim" (seconds >= 0, "segments" mode and the +// transcribe functions; 0 = no trimming), "mode" ("speech" or "segments"), +// "probabilities" (bool), "speech_pad" (seconds >= 0, "speech" mode). With +// `allow_filter` the word filter keys of parse_filter_options are accepted too +// and stored in `req.filter`. Unknown keys and bad values are errors. The +// values that a key leaves out come from default_segmenter_opts(kind). Returns +// false and sets `err` on failure. bool parse_vad_options(const char* json, VadRequest& req, std::string& err, - VadKind kind = VadKind::kHead); + VadKind kind = VadKind::kHead, bool allow_filter = false); + +// The word filter options alone: a flat JSON object, or NULL / "" for a filter +// that is off. Keys: "min_local_conf" (0 <= x <= 1, 0 = off), "local_radius" +// (seconds > 0, default 5), "drop_punct_only" (bool). Unknown keys and bad +// values are errors. Returns false and sets `err` on failure. +bool parse_filter_options(const char* json, WordFilter& filter, std::string& err); // Same document for a Silero VAD model. `pcm` is mono at `sample_rate`, which // must be 16000 or 8000 (the caller resamples anything else); times are on the diff --git a/src/vad_segmenter.cpp b/src/vad_segmenter.cpp index c03cad5..bdd5212 100644 --- a/src/vad_segmenter.cpp +++ b/src/vad_segmenter.cpp @@ -20,7 +20,7 @@ std::vector segment_by_vad(const std::vector& p, double total !(o.max_seg_sec > 2.0 * o.frame_sec) || !std::isfinite(total_sec) || !std::isfinite(o.threshold) || !std::isfinite(o.min_seg_sec) || !std::isfinite(o.min_pause_sec) || !std::isfinite(o.bridge_sec) || - !std::isfinite(o.min_speech_sec) || o.max_seg_sec > kMaxSec || o.min_seg_sec > kMaxSec || + !std::isfinite(o.min_speech_sec) || !std::isfinite(o.trim_sec) || o.max_seg_sec > kMaxSec || o.min_seg_sec > kMaxSec || o.min_pause_sec > kMaxSec || o.bridge_sec > kMaxSec || o.min_speech_sec > kMaxSec) { out.push_back({0.0, total_sec}); return out; @@ -67,8 +67,20 @@ std::vector segment_by_vad(const std::vector& p, double total for_runs(false, [&](int64_t a, int64_t b) { if (b - a >= pause_f) pauses.emplace_back(a, b); }); // Step 3: cut from the front. A segment starting at s may end at a pause. + // With trim_sec > 0 the segment shrinks to its speech plus trim_sec on each + // side (never beyond the cut itself), so the decoder gets little else. auto emit = [&](int64_t a, int64_t b, double end_sec) { - if (has_speech(a, std::min(b, n))) out.push_back({(double)a * fs, end_sec}); + const int64_t be = std::min(b, n); + if (!has_speech(a, be)) return; + double s0 = (double)a * fs, e0 = end_sec; + if (o.trim_sec > 0.0) { + int64_t fa = a, fb = be; + while (fa < fb && !sp[(size_t)fa]) ++fa; + while (fb > fa && !sp[(size_t)(fb - 1)]) --fb; + s0 = std::max(s0, (double)fa * fs - o.trim_sec); + e0 = std::min(e0, (double)fb * fs + o.trim_sec); + } + out.push_back({s0, e0}); }; int64_t s = 0; while (total_sec - (double)s * fs > o.max_seg_sec + 1e-9) { diff --git a/src/vad_segmenter.hpp b/src/vad_segmenter.hpp index 34b546e..a9842bf 100644 --- a/src/vad_segmenter.hpp +++ b/src/vad_segmenter.hpp @@ -18,6 +18,11 @@ struct SegmenterOpts { double bridge_sec = 0.1; // speech gaps shorter than this are bridged double min_speech_sec = 0.1; // speech runs shorter than this are dropped double pad_sec = 0.0; // speech_regions only: padding added on both sides of a region + // segment_by_vad only: a segment cut from long audio shrinks to its first + // and last speech frame plus this many seconds on each side. 0 keeps the + // whole cut (the behaviour before trimming existed). Audio of at most + // max_seg_sec is not affected. + double trim_sec = 0.3; }; // Which model made the probabilities. The two kinds differ in frame period @@ -53,11 +58,17 @@ SegmenterOpts default_segmenter_opts(VadKind kind); // 4. Segments that contain no speech are dropped, including the trailing // remainder. The result is empty when no segment has speech. Kept segments // are ordered and disjoint but need not touch. +// 5. With trim_sec > 0 each kept segment is cut down to [first speech frame - +// trim_sec, end of last speech frame + trim_sec], limited to the segment +// itself. Speech is the smoothed mask of step 1. This keeps the quiet or +// noisy parts that a hard or pause cut leaves at the edges away from the +// decoder. A segment is never trimmed to nothing: it has speech, so it keeps +// at least that run. Trimmed edges are not on the frame grid. // // Degenerate options (frame_sec not finite or <= 0, max_seg_sec not finite or -// <= 2 * frame_sec, threshold or any of the four durations not finite, any -// duration above 1e6 seconds) return the single segment {0, total_sec}. Every -// internal boundary is a whole number of frames. +// <= 2 * frame_sec, threshold, trim_sec or any of the four durations not finite, +// any duration above 1e6 seconds) return the single segment {0, total_sec}. +// Every cut between two segments is a whole number of frames. std::vector segment_by_vad(const std::vector& p, double total_sec, const SegmenterOpts& o); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3c9f63e..c360406 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -18,6 +18,7 @@ pk_add_test(test_model_loader_ternary) pk_add_test(test_ternary_load_negative) pk_add_test(test_vad_head) pk_add_test(test_vad_segmenter) +pk_add_test(test_word_filter) pk_add_test(test_fft) pk_add_test(test_mel) pk_add_test(test_mel_gpu) @@ -249,10 +250,12 @@ pk_add_test(test_vad_options) pk_add_test(test_capi_vad_silero) pk_add_test(test_transcribe_vad_silero) pk_add_test(test_vad_only) +pk_add_test(test_vad_trim_filter) target_compile_definitions(test_silero_vad PRIVATE PK_SOURCE_DIR="${CMAKE_SOURCE_DIR}") set_tests_properties(test_silero_vad test_capi_vad_silero test_transcribe_vad_silero PROPERTIES LABELS "model") set_tests_properties(test_capi_vad_silero test_transcribe_vad_silero PROPERTIES WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) set_tests_properties(test_vad_only PROPERTIES LABELS "model") +set_tests_properties(test_vad_trim_filter PROPERTIES LABELS "model" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) pk_add_test(test_bundle) pk_add_test(test_bundle_models) @@ -261,3 +264,18 @@ if(PARAKEET_WITH_CED AND PARAKEET_WITH_VOICEDETECT) pk_add_test(test_bundle_full) set_tests_properties(test_bundle_full PROPERTIES LABELS "model" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) endif() + +# Option parsing of the trim and word filter flags: each bad value exits at +# once, before any model is read. +if(TARGET parakeet-cli) + function(pk_cli_flag_test name pattern) + add_test(NAME ${name} COMMAND parakeet-cli transcribe --model none.gguf --input none.wav ${ARGN}) + set_tests_properties(${name} PROPERTIES PASS_REGULAR_EXPRESSION "${pattern}") + endfunction() + pk_cli_flag_test(cli_min_local_conf_range "--min-local-conf must be in" --min-local-conf 1.5) + pk_cli_flag_test(cli_min_local_conf_nan "--min-local-conf must be in" --min-local-conf abc) + pk_cli_flag_test(cli_local_radius_zero "--local-radius must be > 0" --local-radius 0) + pk_cli_flag_test(cli_vad_trim_negative "--vad-trim must be >= 0" --vad-trim -1) + pk_cli_flag_test(cli_filter_with_beam "word filter works with greedy decoding only" --min-local-conf 0.5 --beam-size 4) + pk_cli_flag_test(cli_filter_with_stream "word filter is offline only" --drop-punct-only --stream) +endif() diff --git a/tests/test_transcribe_vad.cpp b/tests/test_transcribe_vad.cpp index c63e68d..d587613 100644 --- a/tests/test_transcribe_vad.cpp +++ b/tests/test_transcribe_vad.cpp @@ -72,6 +72,7 @@ int main() { { SegmenterOpts o; o.frame_sec = m->config().vad.frame_sec; + o.trim_sec = 0.0; // the cuts themselves are contiguous; trimming is tested in test_vad_trim_filter const auto segs = segment_by_vad(m->vad_probabilities(clip.samples), total, o); std::printf("62 s clip (%.2f s): %zu segments:", total, segs.size()); for (const auto& sg : segs) std::printf(" [%.2f-%.2f]", sg.start, sg.end); diff --git a/tests/test_vad_batched.cpp b/tests/test_vad_batched.cpp index 6e91b3d..02010d1 100644 --- a/tests/test_vad_batched.cpp +++ b/tests/test_vad_batched.cpp @@ -51,13 +51,16 @@ static void check_model(const char* path, const Audio& clip) { (double)cfg.hop_length * (double)cfg.subsampling_factor / (double)cfg.sample_rate; const double total = (double)clip.samples.size() / 16000.0; + // Trim 0.3 (the default) and 0 (the cuts as they were before trimming). + for (double trim : {0.3, 0.0}) for (double max_seg : {3.0, 7.0}) { SegmenterOpts opts; opts.max_seg_sec = max_seg; + opts.trim_sec = trim; SegmenterOpts so = opts; so.frame_sec = cfg.vad.frame_sec; const std::vector segs = segment_by_vad(m->vad_probabilities(clip.samples), total, so); - std::fprintf(stderr, " max_seg %.0f s: %zu segments\n", max_seg, segs.size()); + std::fprintf(stderr, " trim %.1f max_seg %.0f s: %zu segments\n", trim, max_seg, segs.size()); if (max_seg == 3.0) CHECK(segs.size() > 16); // more than one group of 16 CHECK(!segs.empty()); diff --git a/tests/test_vad_options.cpp b/tests/test_vad_options.cpp index f4eadb6..b7b894c 100644 --- a/tests/test_vad_options.cpp +++ b/tests/test_vad_options.cpp @@ -64,6 +64,59 @@ int main() { } } + // "trim": default 0.3 for both kinds, 0 turns it off, negative is an error. + CHECK(parse(nullptr, VadKind::kHead, r, err) && near(r.opts.trim_sec, 0.3)); + CHECK(parse(nullptr, VadKind::kSilero, r, err) && near(r.opts.trim_sec, 0.3)); + CHECK(parse("{\"trim\":0}", VadKind::kHead, r, err) && r.opts.trim_sec == 0.0 && near(r.opts.max_seg_sec, 30.0)); + CHECK(parse("{\"trim\":0.5,\"mode\":\"segments\"}", VadKind::kSilero, r, err) && near(r.opts.trim_sec, 0.5) && + r.mode == VadRequest::Mode::kSegments && near(r.opts.pad_sec, 0.03)); + for (VadKind k : {VadKind::kHead, VadKind::kSilero}) { + CHECK(!parse("{\"trim\":-0.1}", k, r, err) && err.find("trim") != std::string::npos); + CHECK(!parse("{\"trim\":\"x\"}", k, r, err) && err.find("trim") != std::string::npos); + CHECK(!parse("{\"trim\":1e9}", k, r, err) && err.find("trim") != std::string::npos); + } + + // Word filter keys: only with allow_filter; off by default. + CHECK(parse(nullptr, VadKind::kHead, r, err) && !r.filter.active() && near(r.filter.local_radius_sec, 5.0)); + CHECK(!parse_vad_options("{\"min_local_conf\":0.5}", r, err, VadKind::kHead, false) && + err.find("min_local_conf") != std::string::npos); + CHECK(!parse_vad_options("{\"drop_punct_only\":true}", r, err, VadKind::kHead, false) && + err.find("drop_punct_only") != std::string::npos); + CHECK(parse_vad_options("{\"min_local_conf\":0.5,\"local_radius\":3,\"drop_punct_only\":true,\"trim\":0.2,\"max_segment\":10}", + r, err, VadKind::kSilero, true)); + CHECK(r.filter.min_local_conf == 0.5f && r.filter.local_radius_sec == 3.0f && r.filter.drop_punct_only && + r.filter.active() && near(r.opts.trim_sec, 0.2) && near(r.opts.max_seg_sec, 10.0)); + CHECK(parse_vad_options("{\"min_local_conf\":0,\"drop_punct_only\":false}", r, err, VadKind::kHead, true) && !r.filter.active()); + CHECK(parse_vad_options("{\"min_local_conf\":1}", r, err, VadKind::kHead, true) && r.filter.min_local_conf == 1.0f); + { + const struct { const char* json; const char* word; } badf[] = { + {"{\"min_local_conf\":1.01}", "min_local_conf"}, {"{\"min_local_conf\":-0.5}", "min_local_conf"}, + {"{\"min_local_conf\":\"x\"}", "min_local_conf"}, {"{\"local_radius\":0}", "local_radius"}, + {"{\"local_radius\":-1}", "local_radius"}, {"{\"local_radius\":1e9}", "local_radius"}, + {"{\"drop_punct_only\":1}", "drop_punct_only"}, {"{\"drop_punct_only\":\"true\"}", "drop_punct_only"}, + }; + for (const auto& b : badf) { + CHECK(!parse_vad_options(b.json, r, err, VadKind::kHead, true)); + CHECK(err.find(b.word) != std::string::npos); + } + } + // parse_filter_options: the filter keys alone. + { + WordFilter f; + CHECK(parse_filter_options(nullptr, f, err) && !f.active()); + CHECK(parse_filter_options("", f, err) && !f.active()); + CHECK(parse_filter_options("{}", f, err) && !f.active() && f.local_radius_sec == 5.0f); + CHECK(parse_filter_options("{\"min_local_conf\":0.5}", f, err) && f.min_local_conf == 0.5f && f.local_radius_sec == 5.0f && + !f.drop_punct_only); + CHECK(parse_filter_options("{\"drop_punct_only\":true,\"local_radius\":2.5}", f, err) && f.drop_punct_only && + f.min_local_conf == 0.0f && f.local_radius_sec == 2.5f); + const char* bad_only[] = {"{\"threshold\":0.5}", "{\"trim\":0.3}", "{\"mode\":\"speech\"}", "{\"max_segment\":5}", + "{\"nope\":1}", "[1]", "{\"min_local_conf\":", "{} x", "{\"min_local_conf\":2}"}; + for (const char* j : bad_only) CHECK(!parse_filter_options(j, f, err) && !err.empty()); + CHECK(!parse_filter_options("{\"threshold\":0.5}", f, err) && err.find("threshold") != std::string::npos && + err.find("unknown") != std::string::npos); + } + // C-API: NULL and wrong-kind contexts fail without a crash. CHECK(parakeet_capi_vad_pcm_json(nullptr, nullptr, 0, 16000, nullptr) == nullptr); CHECK(parakeet_capi_vad_path_json(nullptr, "x.wav", nullptr) == nullptr); @@ -72,6 +125,7 @@ int main() { CHECK(parakeet_capi_vad_stream_reset(nullptr) != 0); parakeet_capi_vad_stream_free(nullptr); CHECK(parakeet_capi_transcribe_path_json_vad_with(nullptr, nullptr, "x.wav", 0, nullptr) == nullptr); + CHECK(parakeet_capi_transcribe_path_json_with(nullptr, "x.wav", 0, nullptr) == nullptr); CHECK(parakeet_capi_model_kind(nullptr) == PARAKEET_MODEL_KIND_NONE); CHECK(parakeet_capi_load("does-not-exist.gguf") == nullptr); diff --git a/tests/test_vad_segmenter.cpp b/tests/test_vad_segmenter.cpp index 2eb4536..fb6e875 100644 --- a/tests/test_vad_segmenter.cpp +++ b/tests/test_vad_segmenter.cpp @@ -1,6 +1,7 @@ #include "vad_segmenter.hpp" #include +#include #include #include #include @@ -14,6 +15,12 @@ static int failures = 0; static const double FS = 0.08; +// Options with trimming off: the segmenter before trim_sec existed. +static SegmenterOpts legacy(SegmenterOpts o = SegmenterOpts()) { + o.trim_sec = 0.0; + return o; +} + // n frames of speech (p = 0.95) with silent ranges [a, b) in frames (p = 0.02). static std::vector make_p(int n, std::initializer_list> silences) { std::vector p((size_t)n, 0.95f); @@ -46,10 +53,13 @@ static void test_defaults() { CHECK(near(o.min_seg_sec, 1.0)); CHECK(near(o.bridge_sec, 0.1)); CHECK(near(o.min_speech_sec, 0.1)); + CHECK(near(o.trim_sec, 0.3)); + CHECK(near(default_segmenter_opts(VadKind::kHead).trim_sec, 0.3)); + CHECK(near(default_segmenter_opts(VadKind::kSilero).trim_sec, 0.3)); } static void test_short_is_single() { - SegmenterOpts o; + SegmenterOpts o = legacy(); auto s = segment_by_vad(make_p(250, {}), 20.0, o); CHECK(s.size() == 1 && s[0].start == 0.0 && s[0].end == 20.0); s = segment_by_vad({}, 0.5, o); // shorter than one frame @@ -62,7 +72,7 @@ static void test_short_is_single() { } static void test_threshold_is_inclusive() { - SegmenterOpts o; + SegmenterOpts o = legacy(); // 70 s of p == 0.5 exactly is all speech: hard cuts, nothing dropped. std::vector p(875, 0.5f); auto s = segment_by_vad(p, 70.0, o); @@ -73,7 +83,7 @@ static void test_threshold_is_inclusive() { } static void test_all_speech_hard_cuts() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 70.0; auto s = segment_by_vad(make_p((int)std::ceil(total / FS), {}), total, o); check_ordered(s, total, o.max_seg_sec); @@ -86,7 +96,7 @@ static void test_all_speech_hard_cuts() { } static void test_cuts_land_in_pauses() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 70.0; // pauses at [312,325) and [640,653): midpoints 318 (25.44 s) and 646 (51.68 s) auto p = make_p((int)std::ceil(total / FS), {{312, 325}, {640, 653}}); @@ -102,7 +112,7 @@ static void test_cuts_land_in_pauses() { } static void test_picks_last_pause_not_longest() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 40.0; // A long pause at 10-12 s and a short one (0.4 s) at 27 s. The last one wins. auto p = make_p((int)std::ceil(total / FS), {{125, 150}, {337, 342}}); @@ -113,7 +123,7 @@ static void test_picks_last_pause_not_longest() { } static void test_pause_must_be_fully_inside() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 45.0; // Pause 1 [200,230) is fully inside the window [0, 375]. Pause 2 [370,390) // starts inside but ends after 30 s, so it is not fully inside: pause 1 wins @@ -126,7 +136,7 @@ static void test_pause_must_be_fully_inside() { } static void test_midpoint_fallback() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 45.0; // Only pause: [350,390) = 28.0 to 31.2 s. It is not fully inside the 30 s // window, but its midpoint (370 = 29.6 s) is, so the cut goes there. @@ -143,7 +153,7 @@ static void test_midpoint_fallback() { } static void test_min_segment_one_second() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 31.0; // Pause [6,20): starts before 1 s, midpoint 13 = 1.04 s is inside: cut there. auto p = make_p((int)std::ceil(total / FS), {{6, 20}}); @@ -157,7 +167,7 @@ static void test_min_segment_one_second() { } static void test_min_pause() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 45.0; // 2 frames (0.16 s) is shorter than 0.2 s: hard cut at 30 s. auto s = segment_by_vad(make_p((int)std::ceil(total / FS), {{312, 314}}), total, o); @@ -168,7 +178,7 @@ static void test_min_pause() { } static void test_bridge_short_speech_gap() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 45.0; // A one-frame (0.08 s) dip inside speech is bridged, so it cannot form a // pause even when min_pause is set below one frame. @@ -183,7 +193,7 @@ static void test_bridge_short_speech_gap() { } static void test_drop_short_speech_runs() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 40.0; const int n = (int)std::ceil(total / FS); // Speech 0-5 s, then a lone one-frame blip at 34 s, silence elsewhere. The @@ -206,7 +216,7 @@ static void test_drop_short_speech_runs() { } static void test_segments_without_speech_are_dropped() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 70.0; const int n = (int)std::ceil(total / FS); // Speech for the first 10 s, then nothing. The pause [125,875) is not inside @@ -228,7 +238,7 @@ static void test_segments_without_speech_are_dropped() { } static void test_all_silence() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 65.0; CHECK(segment_by_vad(make_p((int)std::ceil(total / FS), {{0, 100000}}), total, o).empty()); CHECK(segment_by_vad({}, 65.0, o).empty()); @@ -238,7 +248,7 @@ static void test_random_property() { std::mt19937 rng(7); std::uniform_real_distribution u(0.0f, 1.0f); std::uniform_int_distribution run_len(1, 12); - SegmenterOpts o; + SegmenterOpts o = legacy(); for (int trial = 0; trial < 200; ++trial) { const double total = 31.0 + 600.0 * u(rng); const int n = (int)std::ceil(total / FS); @@ -265,13 +275,13 @@ static void test_random_property() { static void test_degenerate_opts() { const double bad[] = {0.0, -0.08, INFINITY, NAN}; for (double fs : bad) { - SegmenterOpts o; + SegmenterOpts o = legacy(); o.frame_sec = fs; auto s = segment_by_vad(std::vector(100, 0.0f), 100.0, o); CHECK(s.size() == 1); if (s.size() == 1) CHECK(s[0].start == 0.0 && s[0].end == 100.0); } - SegmenterOpts o; + SegmenterOpts o = legacy(); o.max_seg_sec = 0.05; // smaller than a frame auto s = segment_by_vad(std::vector(100, 0.0f), 100.0, o); CHECK(s.size() == 1); @@ -285,7 +295,7 @@ static void test_nonfinite_and_huge_opts() { const double bads[] = {INFINITY, -INFINITY, NAN, 1e300, 1e7}; for (double v : bads) { for (int which = 0; which < 5; ++which) { - SegmenterOpts o; + SegmenterOpts o = legacy(); if (which == 0) o.min_seg_sec = v; else if (which == 1) o.min_pause_sec = v; else if (which == 2) o.max_seg_sec = v; @@ -298,7 +308,7 @@ static void test_nonfinite_and_huge_opts() { } const float thr[] = {INFINITY, -INFINITY, NAN}; for (float t : thr) { - SegmenterOpts o; + SegmenterOpts o = legacy(); o.threshold = t; auto s = segment_by_vad(std::vector(2000, 0.0f), 100.0, o); CHECK(s.size() == 1); @@ -309,7 +319,7 @@ static void test_nonfinite_and_huge_opts() { // speech_regions: smoothed speech runs for audio of any length, with gaps // shorter than min_pause merged. static void test_speech_regions() { - SegmenterOpts o; + SegmenterOpts o = legacy(); // 50 frames (4 s): silence 0-10, speech 10-20, silence 20-23 (0.24 s, a // pause), speech 23-35, silence 35-50. std::vector p(50, 0.02f); @@ -371,7 +381,7 @@ static void test_kind_defaults() { // Frames of 32 ms: boundaries are whole frames, a pause is found, caps hold. static void test_segmenter_32ms() { - SegmenterOpts o = default_segmenter_opts(VadKind::kSilero); + SegmenterOpts o = legacy(default_segmenter_opts(VadKind::kSilero)); const double fs = 0.032; // 50 s: speech, a 0.5 s pause at 20 s, speech to the end. const int n = (int)std::ceil(50.0 / fs); @@ -507,6 +517,179 @@ static void test_event_tracker_matches_offline() { CHECK(ev.size() == 2 && ev[0].start && near(ev[1].time, 0.5)); } + +// ---- trim_sec ------------------------------------------------------------ + +// n frames at p = 0.02 with speech (p = 0.95) in the given frame ranges. +static std::vector speech_p(int n, std::initializer_list> speech) { + std::vector p((size_t)n, 0.02f); + for (auto r : speech) + for (int i = r.first; i < r.second && i < n; ++i) p[(size_t)i] = 0.95f; + return p; +} + +static bool seg_is(const std::vector& s, std::initializer_list> want) { + if (s.size() != want.size()) return false; + size_t i = 0; + for (auto w : want) { + if (!near(s[i].start, w.first) || !near(s[i].end, w.second)) return false; + ++i; + } + return true; +} + +// Head frames (0.08 s), default trim 0.3 s. +static void test_trim_head() { + const SegmenterOpts o; // trim_sec 0.3 + // Speech [330,375) = 26.4-30.0 s and [450,500) = 36.0-40.0 s in 45 s. The + // cuts are at the middle of the leading silence (frame 165, which holds no + // speech, so that piece is dropped) and of the pause (frame 412 = 32.96 s). + auto p = speech_p(563, {{330, 375}, {450, 500}}); + auto s = segment_by_vad(p, 45.0, o); + CHECK(seg_is(s, {{26.4 - 0.3, 30.0 + 0.3}, {36.0 - 0.3, 40.0 + 0.3}})); + // trim 0 keeps the whole cuts. + s = segment_by_vad(p, 45.0, legacy()); + CHECK(seg_is(s, {{165 * FS, 412 * FS}, {412 * FS, 45.0}})); + // Speech at both edges of the audio: the start cannot go below 0 and the + // end not beyond the audio. Cut at the middle of the pause (frame 250). + p = speech_p(500, {{0, 10}, {490, 500}}); + s = segment_by_vad(p, 40.0, o); + CHECK(seg_is(s, {{0.0, 0.8 + 0.3}, {39.2 - 0.3, 40.0}})); + // No speech anywhere: nothing is left, and no segment is trimmed to nothing. + CHECK(segment_by_vad(speech_p(875, {}), 70.0, o).empty()); + // Two speech runs 0.16 s apart (not a pause) stay in one segment, and the + // gap between them is kept. The audio after the last run is dropped. + p = speech_p(563, {{100, 120}, {122, 140}}); + s = segment_by_vad(p, 45.0, o); + CHECK(seg_is(s, {{8.0 - 0.3, 11.2 + 0.3}})); + // Two runs far apart, in two segments: each is trimmed on its own, and the + // trim never reaches into the neighbouring segment. + p = speech_p(563, {{20, 40}, {60, 80}}); + s = segment_by_vad(p, 45.0, o); + CHECK(seg_is(s, {{1.6 - 0.3, 3.2 + 0.3}, {4.8 - 0.3, 6.4 + 0.3}})); + // A trim larger than the pause is limited by the cut (cut at frame 50 = 4.0 s). + SegmenterOpts big; + big.trim_sec = 5.0; + s = segment_by_vad(p, 45.0, big); + CHECK(s.size() == 2 && near(s[0].start, 0.0) && near(s[0].end, 4.0) && near(s[1].start, 4.0)); + check_ordered(s, 45.0, 30.0); +} + +// Silero frames (0.032 s). +static void test_trim_silero() { + const SegmenterOpts o = default_segmenter_opts(VadKind::kSilero); + CHECK(near(o.trim_sec, 0.3)); + const double fs = 0.032; + // Speech at the end of a window and a later run. 45 s, n = 1407. + // speech [900,937) and [1200,1250). The first cut is in the leading + // silence (frame 450), the second in the pause (frame 1068). + auto p = speech_p(1407, {{900, 937}, {1200, 1250}}); + auto s = segment_by_vad(p, 45.0, o); + CHECK(seg_is(s, {{900 * fs - 0.3, 937 * fs + 0.3}, {1200 * fs - 0.3, 1250 * fs + 0.3}})); + s = segment_by_vad(p, 45.0, legacy(o)); + CHECK(seg_is(s, {{450 * fs, 1068 * fs}, {1068 * fs, 45.0}})); + // Speech at both edges. 40 s, n = 1250, cut at frame 625 (20 s). + p = speech_p(1250, {{0, 40}, {1210, 1250}}); + s = segment_by_vad(p, 40.0, o); + CHECK(seg_is(s, {{0.0, 40 * fs + 0.3}, {1210 * fs - 0.3, 40.0}})); + // No speech. + CHECK(segment_by_vad(speech_p(1407, {}), 45.0, o).empty()); + // A short run (below min_speech 0.25 s) is no speech and is not kept. + CHECK(segment_by_vad(speech_p(1407, {{500, 507}}), 45.0, o).empty()); +} + +// Speech that runs through a hard cut is not trimmed at the cut, and a clip of +// at most max_seg_sec is returned whole. +static void test_trim_keeps_cuts_and_short_audio() { + const SegmenterOpts o; + auto s = segment_by_vad(speech_p(875, {{0, 875}}), 70.0, o); + CHECK(seg_is(s, {{0.0, 30.0}, {30.0, 60.0}, {60.0, 70.0}})); + s = segment_by_vad(speech_p(300, {{100, 110}}), 24.0, o); // 24 s: whole + CHECK(seg_is(s, {{0.0, 24.0}})); + s = segment_by_vad(speech_p(375, {{100, 110}}), 30.0, o); // exactly the cap + CHECK(seg_is(s, {{0.0, 30.0}})); + // Speech only in the leading 0.1 s of the second half of a hard cut. + s = segment_by_vad(speech_p(875, {{0, 375}, {375, 377}}), 70.0, o); + CHECK(s.size() == 2 && near(s[1].start, 30.0) && near(s[1].end, 30.0 + 0.16 + 0.3)); + // A bad trim is degenerate like the other options. + SegmenterOpts bad; + bad.trim_sec = NAN; + s = segment_by_vad(speech_p(875, {{0, 10}}), 70.0, bad); + CHECK(seg_is(s, {{0.0, 70.0}})); + // A negative trim means off. + bad.trim_sec = -1.0; + s = segment_by_vad(speech_p(563, {{20, 40}, {60, 80}}), 45.0, bad); + CHECK(s.size() == 2 && near(s[0].start, 0.0) && near(s[0].end, 4.0)); +} + +// Random streams: trimmed segments are inside the old cuts, contain the same +// speech, and never exceed trim_sec of non-speech at either edge. +static void test_trim_random_property() { + std::mt19937 rng(11); + for (int trial = 0; trial < 200; ++trial) { + const SegmenterOpts old = legacy(default_segmenter_opts(trial % 2 ? VadKind::kSilero : VadKind::kHead)); + SegmenterOpts nw = old; + nw.trim_sec = 0.1 * (double)(rng() % 10); + const double fs = old.frame_sec; + const int n = (int)(35.0 / fs) + (int)(rng() % 2000); + std::vector p; + bool sp = rng() & 1; + while ((int)p.size() < n) { + const int len = 1 + (int)(rng() % 300); + for (int i = 0; i < len && (int)p.size() < n; ++i) p.push_back(sp ? 0.9f : 0.05f); + sp = !sp; + } + const double total = (double)n * fs; + const auto a = segment_by_vad(p, total, old); + const auto b = segment_by_vad(p, total, nw); + CHECK(a.size() == b.size()); + for (size_t i = 0; i < a.size() && i < b.size(); ++i) { + CHECK(b[i].start >= a[i].start - 1e-9 && b[i].end <= a[i].end + 1e-9); + CHECK(b[i].end > b[i].start); + } + check_ordered(b, total, old.max_seg_sec); + } +} + +// With trim_sec 0 the output equals what the segmenter gave before trimming +// existed: the digest was computed with the previous implementation on the same +// seeded streams (80 ms and 32 ms frames, assorted options). +// Digest of segment_by_vad over seeded random inputs (FNV-1a over the printed segments). +static uint64_t digest_segments() { + std::mt19937 rng(20261004); + uint64_t h = 1469598103934665603ull; + auto mix = [&](const char* s) { for (; *s; ++s) { h ^= (unsigned char)*s; h *= 1099511628211ull; } }; + for (int kind = 0; kind < 2; ++kind) { + for (int iter = 0; iter < 150; ++iter) { + SegmenterOpts o = default_segmenter_opts(kind ? VadKind::kSilero : VadKind::kHead); + o.trim_sec = 0.0; + if (iter % 3 == 1) o.min_pause_sec = 0.05 + 0.01 * (double)(rng() % 40); + if (iter % 4 == 2) o.max_seg_sec = 5.0 + (double)(rng() % 250) / 10.0; + const int n = (int)(o.max_seg_sec / o.frame_sec) + 1 + (int)(rng() % 4000); + std::vector p; + bool sp = rng() & 1; + while ((int)p.size() < n) { + const int len = 1 + (int)(rng() % (iter % 2 ? 40 : 400)); + for (int i = 0; i < len && (int)p.size() < n; ++i) p.push_back(sp ? 0.9f : 0.05f); + sp = !sp; + } + const double total = (double)n * o.frame_sec - (double)(rng() % 100) / 100.0 * o.frame_sec * 0.9; + char buf[96]; + std::snprintf(buf, sizeof(buf), "#%d,%d;", kind, iter); + mix(buf); + for (const VadSegment& g : segment_by_vad(p, total, o)) { + std::snprintf(buf, sizeof(buf), "%.9f,%.9f;", g.start, g.end); + mix(buf); + } + } + } + return h; +} + +static void test_trim_zero_equals_previous_output() { + CHECK(digest_segments() == 16100934503807512709ull); +} + int main() { test_defaults(); test_nonfinite_and_huge_opts(); @@ -530,6 +713,11 @@ int main() { test_segmenter_32ms(); test_silero_speech_regions_and_pad(); test_event_tracker_matches_offline(); + test_trim_head(); + test_trim_silero(); + test_trim_keeps_cuts_and_short_audio(); + test_trim_random_property(); + test_trim_zero_equals_previous_output(); if (failures) return 1; std::puts("test_vad_segmenter: OK"); return 0; diff --git a/tests/test_vad_trim_filter.cpp b/tests/test_vad_trim_filter.cpp new file mode 100644 index 0000000..aabf88b --- /dev/null +++ b/tests/test_vad_trim_filter.cpp @@ -0,0 +1,331 @@ +// VAD segment trimming and the word filter on real models. +// +// Clip: fixtures joined by long stretches of digital silence, so the VAD finds +// pauses. Checks, per model: +// * trimmed segments carry less audio, and the words and their times match the +// untrimmed decode of the same speech (offsets are right); +// * trim 0 gives the same Transcription as before trimming existed (the +// segments of the legacy segmenter, decoded one by one); +// * the word filter off is byte-identical: Model call, C-API JSON; +// * the filter on: guard member, words removed from text, words and tokens. +// Env (each optional; skip 77 when none is set): +// PARAKEET_TEST_GGUF_ULTRA / _REDUX_KEEP / _REDUX_DEQ models with a VAD head +// PARAKEET_TEST_GGUF any ASR model (with PARAKEET_TEST_SILERO_GGUF: Silero cuts) +// PARAKEET_TEST_GGUF_CTC a CTC model: the filter keys on the CTC head +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "audio_io.hpp" +#include "model.hpp" +#include "parakeet_capi.h" +#include "silero_vad.hpp" + +using namespace pk; + +static int failures = 0; +#define CHECK(c) do { if (!(c)) { std::fprintf(stderr, "FAIL: %s (line %d)\n", #c, __LINE__); ++failures; } } while (0) + +static std::vector words_of(const std::string& s) { + std::istringstream is(s); + std::vector w; + for (std::string x; is >> x;) { + std::string y; + for (char c : x) if (std::isalnum((unsigned char)c)) y += (char)std::tolower((unsigned char)c); + if (!y.empty()) w.push_back(y); + } + return w; +} + +static bool same_f(float a, float b) { return std::memcmp(&a, &b, sizeof(float)) == 0; } +static bool same_tr(const Transcription& a, const Transcription& b) { + if (a.text != b.text || a.words.size() != b.words.size() || a.tokens.size() != b.tokens.size() || + a.dropped_words != b.dropped_words) + return false; + for (size_t i = 0; i < a.words.size(); ++i) + if (a.words[i].text != b.words[i].text || !same_f(a.words[i].start, b.words[i].start) || + !same_f(a.words[i].end, b.words[i].end) || !same_f(a.words[i].conf, b.words[i].conf)) + return false; + for (size_t i = 0; i < a.tokens.size(); ++i) + if (a.tokens[i].id != b.tokens[i].id || a.tokens[i].frame != b.tokens[i].frame || + a.tokens[i].span != b.tokens[i].span || !same_f(a.tokens[i].conf, b.tokens[i].conf)) + return false; + return true; +} + +// speech.wav, 24 s of digital silence, speech.wav, 24 s of silence: about 62 s. +static Audio make_clip(double* first_end, double* second_start) { + Audio sp; + CHECK(load_audio_16k_mono("tests/fixtures/speech.wav", sp)); + Audio clip; + clip.sample_rate = 16000; + const std::vector gap((size_t)(24 * 16000), 0.0f); + clip.samples.insert(clip.samples.end(), sp.samples.begin(), sp.samples.end()); + *first_end = (double)clip.samples.size() / 16000.0; + clip.samples.insert(clip.samples.end(), gap.begin(), gap.end()); + *second_start = (double)clip.samples.size() / 16000.0; + clip.samples.insert(clip.samples.end(), sp.samples.begin(), sp.samples.end()); + clip.samples.insert(clip.samples.end(), gap.begin(), gap.end()); + return clip; +} + +static size_t edit_distance(const std::vector& a, const std::vector& b) { + std::vector prev(b.size() + 1), cur(b.size() + 1); + for (size_t j = 0; j <= b.size(); ++j) prev[j] = j; + for (size_t i = 1; i <= a.size(); ++i) { + cur[0] = i; + for (size_t j = 1; j <= b.size(); ++j) + cur[j] = std::min({prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (a[i - 1] == b[j - 1] ? 0 : 1)}); + std::swap(prev, cur); + } + return prev[b.size()]; +} + +static double covered(const std::vector& s) { + double t = 0.0; + for (const auto& g : s) t += g.end - g.start; + return t; +} + +// `fn` set: Silero probabilities (opts are Silero's); else the model's own head. +static void check_trim(const Model& m, const Audio& clip, double first_end, double second_start, + SegmenterOpts opts, const Model::VadProbabilityFn* fn, const char* what, bool strict, + bool quiet_gap) { + std::fprintf(stderr, " %s: trim\n", what); + const double total = (double)clip.samples.size() / 16000.0; + SegmenterOpts so = opts; + if (!fn) so.frame_sec = m.config().vad.frame_sec; + const std::vector p = fn ? (*fn)(clip.samples) : m.vad_probabilities(clip.samples); + SegmenterOpts old = so; + old.trim_sec = 0.0; + const auto segs_new = segment_by_vad(p, total, so); + const auto segs_old = segment_by_vad(p, total, old); + std::fprintf(stderr, " %zu segments, %.1f s with trim, %.1f s without\n", segs_new.size(), + covered(segs_new), covered(segs_old)); + CHECK(segs_new.size() >= 2 && segs_new.size() == segs_old.size()); + CHECK(covered(segs_new) < covered(segs_old)); + // The detector calls the digital silence between the speech non-speech: only + // the speech is decoded. (The Redux head fires on digital silence, so it does + // not get this check.) + if (quiet_gap) { + CHECK(covered(segs_new) < covered(segs_old) - 10.0); + CHECK(covered(segs_new) < 2.0 * first_end + 4 * so.trim_sec + 1.0); + } + + // Word times: the second copy of the speech is the first one shifted. + opts.trim_sec = so.trim_sec; + const Transcription tr = m.transcribe_pcm_vad_with_timestamps(clip.samples, 16000, Decoder::kDefault, "", opts, fn); + opts.trim_sec = 0.0; + const Transcription tr0 = m.transcribe_pcm_vad_with_timestamps(clip.samples, 16000, Decoder::kDefault, "", opts, fn); + CHECK(tr.dropped_words == -1 && tr0.dropped_words == -1); + CHECK(!tr.words.empty()); + // Another cut changes the audio the decoder sees, so a model may differ by a + // word or two; the head model with these fixtures does not. + if (strict) { + CHECK(words_of(tr.text) == words_of(tr0.text)); + CHECK(tr.words.size() == tr0.words.size()); + } else { + const auto a = words_of(tr.text), b = words_of(tr0.text); + CHECK(edit_distance(a, b) <= 1 + b.size() / 10); + } + float prev = -1.0f; + for (const Word& w : tr.words) { + CHECK(w.start >= prev - 1e-3f && w.end >= w.start && w.end <= (float)total + 0.01f); + prev = w.start; + } + for (size_t i = 1; i < tr.tokens.size(); ++i) CHECK(tr.tokens[i].frame >= tr.tokens[i - 1].frame); + for (size_t i = 0; strict && i < tr.words.size() && i < tr0.words.size(); ++i) { + // Trimming moves the start of a segment, and with it the 80 ms frame grid + // of the encoder: times agree to a few frames. + CHECK(std::fabs(tr.words[i].start - tr0.words[i].start) < 0.5f); + } + // The first words are in the first speech, the last words in the second. + if (!tr.words.empty()) { + CHECK(tr.words.front().start < (float)first_end); + CHECK(tr.words.back().start > (float)second_start); + size_t in_gap = 0; + for (const Word& w : tr.words) in_gap += (w.start > (float)first_end + 0.5f && w.start < (float)second_start - 0.5f); + if (quiet_gap) CHECK(in_gap == 0); + } + // The text path matches the timestamp path. + CHECK(m.transcribe_pcm_vad(clip.samples, 16000, Decoder::kDefault, "", opts, fn) == tr0.text); +} + +static void check_filter(const Model& m, const char* wav_path, const Audio& clip, const SegmenterOpts& opts, + const Model::VadProbabilityFn* fn, const char* what) { + std::fprintf(stderr, " %s: filter\n", what); + // Off: the filter object changes nothing, on a short clip and on a long one. + Audio sp; + CHECK(load_audio_16k_mono("tests/fixtures/speech.wav", sp)); + { + const Transcription plain = m.transcribe_with_timestamps(sp.samples, 16000, Decoder::kDefault); + const Transcription off = m.transcribe_pcm_vad_with_timestamps(sp.samples, 16000, Decoder::kDefault, "", opts, fn, WordFilter()); + CHECK(same_tr(plain, off) && off.dropped_words == -1); + } + const Transcription base = m.transcribe_pcm_vad_with_timestamps(clip.samples, 16000, Decoder::kDefault, "", opts, fn); + CHECK(same_tr(base, m.transcribe_pcm_vad_with_timestamps(clip.samples, 16000, Decoder::kDefault, "", opts, fn, WordFilter()))); + CHECK(!base.words.empty()); + + // A mild filter keeps clean speech as it is; the guard member says nothing was dropped. + WordFilter mild; + mild.min_local_conf = 0.5f; + const Transcription kept = m.transcribe_pcm_vad_with_timestamps(clip.samples, 16000, Decoder::kDefault, "", opts, fn, mild); + CHECK(kept.dropped_words == 0); + Transcription expect = base; + expect.dropped_words = 0; + CHECK(same_tr(kept, expect)); + // The text path with a filter gives the filtered text. + CHECK(m.transcribe_pcm_vad(clip.samples, 16000, Decoder::kDefault, "", opts, fn, mild) == kept.text); + + // A threshold of 1 drops every word (no word has mean confidence 1), per + // decode unit, and the count adds up. + WordFilter all; + all.min_local_conf = 1.0f; + const Transcription none = m.transcribe_pcm_vad_with_timestamps(clip.samples, 16000, Decoder::kDefault, "", opts, fn, all); + CHECK(none.words.empty() && none.tokens.empty() && none.text.empty()); + CHECK(none.dropped_words == (int)base.words.size()); + // A radius too small to reach a neighbour leaves the own confidence: the + // low words go, the rest stays, and the numbers agree between words and tokens. + WordFilter own; + own.min_local_conf = 0.9f; + own.local_radius_sec = 0.001f; + const Transcription some = m.transcribe_pcm_vad_with_timestamps(clip.samples, 16000, Decoder::kDefault, "", opts, fn, own); + CHECK(some.dropped_words >= 0 && (size_t)some.dropped_words + some.words.size() == base.words.size()); + CHECK(some.tokens.size() <= base.tokens.size()); + for (const Word& w : some.words) CHECK(w.conf >= 0.9f - 1e-6f); + CHECK(words_of(some.text).size() <= base.words.size()); + (void)wav_path; +} + +static char* take_dup(char* p, std::string& out) { + if (p) { out = p; parakeet_capi_free_string(p); } + return p; +} + +static void check_capi(const char* asr_path, const char* silero_path, const char* what) { + std::fprintf(stderr, " %s: C-API\n", what); + parakeet_ctx* ctx = parakeet_capi_load(asr_path); + CHECK(ctx != nullptr); + if (!ctx) return; + const char* wav = "tests/fixtures/speech.wav"; + std::string plain, off, none_opts, kept, dropped; + CHECK(take_dup(parakeet_capi_transcribe_path_json(ctx, wav, 0), plain)); + // No options, empty options, and an explicit off give the plain document, byte for byte. + CHECK(take_dup(parakeet_capi_transcribe_path_json_with(ctx, wav, 0, nullptr), off) && off == plain); + CHECK(take_dup(parakeet_capi_transcribe_path_json_with(ctx, wav, 0, ""), none_opts) && none_opts == plain); + CHECK(take_dup(parakeet_capi_transcribe_path_json_with(ctx, wav, 0, "{}"), none_opts) && none_opts == plain); + CHECK(take_dup(parakeet_capi_transcribe_path_json_with(ctx, wav, 0, "{\"min_local_conf\":0,\"local_radius\":3}"), none_opts) && + none_opts == plain); + CHECK(plain.find("\"guard\"") == std::string::npos); + // On: guard present; 0.5 keeps clean speech; 1 drops every word. + CHECK(take_dup(parakeet_capi_transcribe_path_json_with(ctx, wav, 0, "{\"min_local_conf\":0.5}"), kept)); + CHECK(kept.size() > plain.size() && kept.find("\"guard\":{\"dropped_words\":0}") != std::string::npos); + CHECK(kept.substr(0, plain.size() - 1) == plain.substr(0, plain.size() - 1)); + CHECK(take_dup(parakeet_capi_transcribe_path_json_with(ctx, wav, 0, "{\"min_local_conf\":1}"), dropped)); + CHECK(dropped.find("\"text\":\"\"") != std::string::npos && dropped.find("\"words\":[]") != std::string::npos && + dropped.find("\"tokens\":[]") != std::string::npos && dropped.find("\"guard\":{\"dropped_words\":") != std::string::npos && + dropped.find("\"dropped_words\":0}") == std::string::npos); + // Errors: NULL, bad keys, bad values, the last error says which. + CHECK(parakeet_capi_transcribe_path_json_with(nullptr, wav, 0, nullptr) == nullptr); + CHECK(parakeet_capi_transcribe_path_json_with(ctx, nullptr, 0, nullptr) == nullptr); + CHECK(parakeet_capi_transcribe_path_json_with(ctx, "/nonexistent.wav", 0, nullptr) == nullptr); + const struct { const char* json; const char* word; } bad[] = { + {"{\"min_local_conf\":1.5}", "min_local_conf"}, {"{\"min_local_conf\":-0.1}", "min_local_conf"}, + {"{\"local_radius\":0}", "local_radius"}, {"{\"drop_punct_only\":1}", "drop_punct_only"}, + {"{\"threshold\":0.5}", "threshold"}, // a VAD key is not a filter key + {"{\"nope\":1}", "nope"}, {"[1]", "JSON object"}}; + for (const auto& b : bad) { + CHECK(parakeet_capi_transcribe_path_json_with(ctx, wav, 0, b.json) == nullptr); + CHECK(std::string(parakeet_capi_last_error(ctx)).find(b.word) != std::string::npos); + } + // The VAD variant takes the same filter keys and the trim key. + if (silero_path) { + parakeet_ctx* v = parakeet_capi_load(silero_path); + CHECK(v != nullptr); + if (v) { + std::string a, b, c; + CHECK(take_dup(parakeet_capi_transcribe_path_json_vad_with(ctx, v, wav, 0, nullptr), a)); + CHECK(take_dup(parakeet_capi_transcribe_path_json_vad_with(ctx, v, wav, 0, "{\"trim\":0,\"max_segment\":10}"), b)); + CHECK(take_dup(parakeet_capi_transcribe_path_json_vad_with(ctx, v, wav, 0, "{\"trim\":0.1,\"max_segment\":10,\"min_local_conf\":0.5,\"drop_punct_only\":true}"), c)); + CHECK(a.find("\"guard\"") == std::string::npos && b.find("\"guard\"") == std::string::npos && + c.find("\"guard\":{\"dropped_words\":") != std::string::npos); + CHECK(parakeet_capi_transcribe_path_json_vad_with(ctx, v, wav, 0, "{\"trim\":-1}") == nullptr); + CHECK(std::string(parakeet_capi_last_error(ctx)).find("trim") != std::string::npos); + CHECK(parakeet_capi_transcribe_path_json_vad_with(ctx, v, wav, 0, "{\"min_local_conf\":2}") == nullptr); + parakeet_capi_free(v); + } + } + parakeet_capi_free(ctx); +} + +int main() { + // The Redux head fires on digital silence, so only Ultra gets the "silence is not decoded" checks. + std::vector> heads; + for (const char* e : {"PARAKEET_TEST_GGUF_ULTRA", "PARAKEET_TEST_GGUF_REDUX_KEEP", "PARAKEET_TEST_GGUF_REDUX_DEQ"}) + if (const char* v = std::getenv(e)) heads.push_back({v, std::strstr(e, "ULTRA") != nullptr}); + const char* any = std::getenv("PARAKEET_TEST_GGUF"); + const char* silero = std::getenv("PARAKEET_TEST_SILERO_GGUF"); + const char* ctc = std::getenv("PARAKEET_TEST_GGUF_CTC"); + if (heads.empty() && !(any && silero) && !ctc) { std::puts("skip: no model env set"); return 77; } + + double first_end = 0.0, second_start = 0.0; + const Audio clip = make_clip(&first_end, &second_start); + + for (const auto& h : heads) { + const char* path = h.first; + std::fprintf(stderr, "head model %s\n", path); + std::unique_ptr m = Model::load(path); + if (!m) { CHECK(false); continue; } + if (!m->config().vad.present) continue; + check_trim(*m, clip, first_end, second_start, SegmenterOpts(), nullptr, "head", true, h.second); + check_filter(*m, "", clip, SegmenterOpts(), nullptr, "head"); + check_capi(path, silero, "head"); + } + if (any && silero) { + std::fprintf(stderr, "silero cuts, model %s\n", any); + std::string err; + std::unique_ptr sv = SileroVad::load(silero, &err); + std::unique_ptr m = Model::load(any); + CHECK(sv && m); + if (sv && m) { + const SileroVad* svp = sv.get(); + const Model::VadProbabilityFn fn = [svp](const std::vector& pcm) { + return svp->probabilities(pcm.data(), pcm.size(), 16000); + }; + const SegmenterOpts so = default_segmenter_opts(VadKind::kSilero); + check_trim(*m, clip, first_end, second_start, so, &fn, "silero", false, true); + check_filter(*m, "", clip, so, &fn, "silero"); + check_capi(any, silero, "silero"); + } + } + if (ctc) { + std::fprintf(stderr, "CTC model %s\n", ctc); + std::unique_ptr m = Model::load(ctc); + CHECK(m != nullptr); + if (m) { + Audio sp; + CHECK(load_audio_16k_mono("tests/fixtures/speech.wav", sp)); + Transcription t = m->transcribe_with_timestamps(sp.samples, 16000, Decoder::kCTC); + const Transcription ref = t; + WordFilter f; + f.min_local_conf = 0.5f; + f.drop_punct_only = true; + apply_word_filter(t, f); + CHECK(t.dropped_words == 0 && t.text == ref.text && t.words.size() == ref.words.size()); + f.min_local_conf = 1.0f; + apply_word_filter(t, f); + CHECK(t.words.empty() && t.tokens.empty() && t.dropped_words == (int)ref.words.size()); + } + } + if (failures) return 1; + std::puts("test_vad_trim_filter: OK"); + return 0; +} diff --git a/tests/test_word_filter.cpp b/tests/test_word_filter.cpp new file mode 100644 index 0000000..fd1c513 --- /dev/null +++ b/tests/test_word_filter.cpp @@ -0,0 +1,200 @@ +// pk::apply_word_filter on scripted words and confidences (no model). +#include "transcription.hpp" + +#include +#include +#include +#include +#include + +using namespace pk; + +static int failures = 0; +#define CHECK(c) do { if (!(c)) { std::fprintf(stderr, "FAIL: %s (line %d)\n", #c, __LINE__); ++failures; } } while (0) + +struct W { const char* text; float start; float conf; }; + +// One token per word: token i has id i and belongs to word i. +static Transcription make(const std::vector& ws) { + Transcription t; + for (size_t i = 0; i < ws.size(); ++i) { + Word w; + w.text = ws[i].text; + w.start = ws[i].start; + w.end = w.start + 0.2f; + w.conf = ws[i].conf; + w.tok_first = w.tok_last = (int32_t)i; + t.words.push_back(w); + t.tokens.push_back(TokenInfo{(int32_t)i, (int32_t)(w.start / 0.08f), w.conf, 1}); + if (!t.text.empty()) t.text += ' '; + t.text += w.text; + } + return t; +} + +static bool same(const Transcription& a, const Transcription& b) { + if (a.text != b.text || a.words.size() != b.words.size() || a.tokens.size() != b.tokens.size() || + a.dropped_words != b.dropped_words) return false; + for (size_t i = 0; i < a.words.size(); ++i) { + const Word &x = a.words[i], &y = b.words[i]; + if (x.text != y.text || x.start != y.start || x.end != y.end || x.conf != y.conf || + x.tok_first != y.tok_first || x.tok_last != y.tok_last) return false; + } + for (size_t i = 0; i < a.tokens.size(); ++i) + if (a.tokens[i].id != b.tokens[i].id || a.tokens[i].frame != b.tokens[i].frame || + a.tokens[i].conf != b.tokens[i].conf || a.tokens[i].span != b.tokens[i].span) return false; + return true; +} + +static void test_off_is_identity() { + const Transcription ref = make({{"a", 0.f, 0.9f}, {"b", 1.f, 0.05f}, {".", 2.f, 0.1f}}); + Transcription t = ref; + CHECK(!WordFilter().active()); + CHECK(WordFilter().min_local_conf == 0.0f && WordFilter().local_radius_sec == 5.0f && !WordFilter().drop_punct_only); + CHECK(apply_word_filter(t, WordFilter()) == 0); + CHECK(same(t, ref) && t.dropped_words == -1); + // A radius alone does not turn the filter on. + WordFilter f; + f.local_radius_sec = 1.0f; + CHECK(!f.active() && apply_word_filter(t, f) == 0 && same(t, ref)); + // Empty input. + Transcription e; + f.min_local_conf = 0.5f; + CHECK(apply_word_filter(e, f) == 0 && e.words.empty() && e.dropped_words == -1); +} + +static void test_isolated_low_word_between_confident_words_is_kept() { + Transcription t = make({{"a", 0.0f, 1.0f}, {"b", 0.3f, 0.2f}, {"c", 0.6f, 1.0f}}); + WordFilter f; + f.min_local_conf = 0.5f; + CHECK(apply_word_filter(t, f) == 0); + CHECK(t.words.size() == 3 && t.text == "a b c" && t.tokens.size() == 3); + CHECK(t.dropped_words == 0); // the filter ran and dropped nothing +} + +static void test_run_of_low_words_is_dropped() { + // Speech, then a run of low confidence words 20 s later, then speech. + Transcription t = make({{"hello", 0.0f, 0.95f}, {"world", 0.5f, 0.9f}, + {"x", 20.0f, 0.2f}, {"y", 20.4f, 0.3f}, {"z", 20.8f, 0.25f}, + {"again", 40.0f, 0.9f}}); + WordFilter f; + f.min_local_conf = 0.5f; + CHECK(apply_word_filter(t, f) == 3); + CHECK(t.dropped_words == 3); + CHECK(t.text == "hello world again"); + CHECK(t.words.size() == 3 && t.tokens.size() == 3); + CHECK(t.tokens[0].id == 0 && t.tokens[1].id == 1 && t.tokens[2].id == 5); + CHECK(t.words[2].text == "again" && t.words[2].tok_first == 2 && t.words[2].tok_last == 2); + // A lone low confidence word with no neighbour in the radius is dropped too. + t = make({{"a", 0.0f, 0.9f}, {"lone", 30.0f, 0.4f}}); + CHECK(apply_word_filter(t, f) == 1 && t.text == "a"); +} + +static void test_radius_and_boundaries() { + WordFilter f; + f.min_local_conf = 0.5f; + f.local_radius_sec = 5.0f; + // B is exactly 5 s after A: inside the window (the bound is inclusive), so + // B's mean is 0.6 and it stays. + Transcription t = make({{"A", 0.0f, 1.0f}, {"B", 5.0f, 0.2f}}); + CHECK(apply_word_filter(t, f) == 0); + // Just under 5 s of radius: B is alone, mean 0.2, dropped. A keeps its own 1.0. + f.local_radius_sec = 4.99f; + t = make({{"A", 0.0f, 1.0f}, {"B", 5.0f, 0.2f}}); + CHECK(apply_word_filter(t, f) == 1 && t.text == "A"); + // The window looks both ways: a low word followed by confident words stays. + f.local_radius_sec = 5.0f; + t = make({{"B", 0.0f, 0.2f}, {"A", 1.0f, 1.0f}, {"C", 2.0f, 1.0f}}); + CHECK(apply_word_filter(t, f) == 0); + // A threshold equal to the mean keeps the words (strictly below is dropped). + f.min_local_conf = 0.75f; + t = make({{"A", 0.0f, 1.0f}, {"B", 1.0f, 0.5f}}); // both have mean 0.75 + CHECK(apply_word_filter(t, f) == 0); + f.min_local_conf = 0.76f; + t = make({{"A", 0.0f, 1.0f}, {"B", 1.0f, 0.5f}}); + CHECK(apply_word_filter(t, f) == 2 && t.words.empty() && t.text.empty() && t.tokens.empty()); + // Words out of time order give the same answer as sorted ones. + f.min_local_conf = 0.5f; + f.local_radius_sec = 2.0f; + t = make({{"x", 20.0f, 0.2f}, {"a", 0.0f, 0.9f}, {"y", 20.5f, 0.3f}, {"b", 0.5f, 0.9f}}); + CHECK(apply_word_filter(t, f) == 2 && t.text == "a b"); +} + +static void test_punct_only() { + WordFilter f; + f.drop_punct_only = true; + CHECK(f.active()); + Transcription t = make({{"a.", 0.0f, 0.9f}, {".", 1.0f, 0.9f}, {"?!", 2.0f, 0.9f}, {"...", 3.0f, 0.9f}, + {"\xE2\x80\xA6", 4.0f, 0.9f}, // ellipsis + {"\xE2\x80\x94", 5.0f, 0.9f}, // em dash + {"\xC2\xBF", 6.0f, 0.9f}, // inverted question mark + {"1", 7.0f, 0.9f}, {"\xC3\xA9", 8.0f, 0.9f}, // e acute is a letter + {"\xE6\x97\xA5", 9.0f, 0.9f}, // a CJK letter + {"-5", 10.0f, 0.9f}}); + CHECK(apply_word_filter(t, f) == 6); + CHECK(t.words.size() == 5); + CHECK(t.text == "a. 1 \xC3\xA9 \xE6\x97\xA5 -5"); + CHECK(t.tokens.size() == 5 && t.tokens[1].id == 7 && t.words[1].tok_first == 1); + // The local filter does not touch confident punctuation, and punct-only + // words count in the mean (that is the input the decoder gave). + f.drop_punct_only = false; + f.min_local_conf = 0.5f; + t = make({{".", 0.0f, 0.9f}, {"a", 1.0f, 0.9f}}); + CHECK(apply_word_filter(t, f) == 0); +} + +static void test_both_checks_together_and_accumulate() { + WordFilter f; + f.min_local_conf = 0.5f; + f.drop_punct_only = true; + Transcription t = make({{"a", 0.0f, 0.9f}, {".", 0.5f, 0.9f}, {"n", 30.0f, 0.1f}}); + CHECK(apply_word_filter(t, f) == 2 && t.text == "a" && t.dropped_words == 2); + // A second call on the same transcription adds to the count. + Word q; + q.text = "q"; q.start = 30.0f; q.end = 30.2f; q.conf = 0.1f; q.tok_first = q.tok_last = (int32_t)t.tokens.size(); + t.words.push_back(q); + t.tokens.push_back(TokenInfo{9, 12, 0.1f, 1}); + t.text += " q"; + CHECK(apply_word_filter(t, f) == 1 && t.dropped_words == 3 && t.text == "a"); +} + +static void test_group_words_records_token_ranges() { + // "hello", "world," (two pieces and a comma), "hi". + const std::vector pieces = {"\xE2\x96\x81hello", "\xE2\x96\x81wor", "ld", ",", "\xE2\x96\x81hi"}; + std::vector tokens; + for (int i = 0; i < 5; ++i) tokens.push_back(TokenInfo{i, i * 2, 0.9f, 1}); + const std::vector ws = group_words(tokens, pieces, 0.08f); + CHECK(ws.size() == 3); + if (ws.size() == 3) { + CHECK(ws[0].text == "hello" && ws[0].tok_first == 0 && ws[0].tok_last == 0); + CHECK(ws[1].text == "world," && ws[1].tok_first == 1 && ws[1].tok_last == 3); + CHECK(ws[2].text == "hi" && ws[2].tok_first == 4 && ws[2].tok_last == 4); + } + // Dropping the middle word removes its three tokens and renumbers the last. + Transcription t; + t.words = ws; + t.tokens = tokens; + t.words[1].conf = 0.05f; + t.words[0].start = 0.0f; t.words[1].start = 30.0f; t.words[2].start = 60.0f; + for (Word& w : t.words) { w.conf = 0.9f; } + t.words[1].conf = 0.05f; + t.text = "hello world, hi"; + WordFilter f; + f.min_local_conf = 0.5f; + CHECK(apply_word_filter(t, f) == 1); + CHECK(t.text == "hello hi" && t.tokens.size() == 2 && t.tokens[0].id == 0 && t.tokens[1].id == 4); + CHECK(t.words[1].tok_first == 1 && t.words[1].tok_last == 1); +} + +int main() { + test_off_is_identity(); + test_isolated_low_word_between_confident_words_is_kept(); + test_run_of_low_words_is_dropped(); + test_radius_and_boundaries(); + test_punct_only(); + test_both_checks_together_and_accumulate(); + test_group_words_records_token_ranges(); + if (failures) return 1; + std::puts("test_word_filter: OK"); + return 0; +}