From 36d4e7e7b1f3cd26d96df16b46a900a6b9efeada Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 4 Oct 2026 20:27:40 +0000 Subject: [PATCH 1/4] feat: trim VAD segments to their speech plus 0.3 s segment_by_vad cuts long audio at pauses, or at the 30 s cap when there is no pause. A cut piece carried everything between its cut points to the decoder, including long stretches of noise and silence that the VAD had already flagged as non-speech. That is where an ASR model invents words. Add SegmenterOpts::trim_sec (default 0.3). Each kept segment shrinks to its first speech frame minus trim_sec and its last speech frame plus trim_sec, limited to the segment itself. Speech is the smoothed mask, so the existing rules still decide what counts: a piece without speech is still dropped, and a piece with speech keeps at least that run. Audio of at most max_seg_sec is not cut and not trimmed. Trim 0 gives the previous output exactly. The change applies to every caller of the segmenter: the Ultra and Redux head, Silero, and the VAD-only slices. Transcripts of long audio through the VAD paths can shift slightly. Tests use synthetic probability streams at 0.08 s and 0.032 s frames with exact expected bounds, and a digest of the previous implementation over seeded random streams to check that trim 0 is unchanged. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- src/vad_segmenter.cpp | 16 ++- src/vad_segmenter.hpp | 17 ++- tests/test_transcribe_vad.cpp | 1 + tests/test_vad_batched.cpp | 5 +- tests/test_vad_segmenter.cpp | 228 +++++++++++++++++++++++++++++++--- 5 files changed, 241 insertions(+), 26 deletions(-) diff --git a/src/vad_segmenter.cpp b/src/vad_segmenter.cpp index c03cad5..bdd5212 100644 --- a/src/vad_segmenter.cpp +++ b/src/vad_segmenter.cpp @@ -20,7 +20,7 @@ std::vector segment_by_vad(const std::vector& p, double total !(o.max_seg_sec > 2.0 * o.frame_sec) || !std::isfinite(total_sec) || !std::isfinite(o.threshold) || !std::isfinite(o.min_seg_sec) || !std::isfinite(o.min_pause_sec) || !std::isfinite(o.bridge_sec) || - !std::isfinite(o.min_speech_sec) || o.max_seg_sec > kMaxSec || o.min_seg_sec > kMaxSec || + !std::isfinite(o.min_speech_sec) || !std::isfinite(o.trim_sec) || o.max_seg_sec > kMaxSec || o.min_seg_sec > kMaxSec || o.min_pause_sec > kMaxSec || o.bridge_sec > kMaxSec || o.min_speech_sec > kMaxSec) { out.push_back({0.0, total_sec}); return out; @@ -67,8 +67,20 @@ std::vector segment_by_vad(const std::vector& p, double total for_runs(false, [&](int64_t a, int64_t b) { if (b - a >= pause_f) pauses.emplace_back(a, b); }); // Step 3: cut from the front. A segment starting at s may end at a pause. + // With trim_sec > 0 the segment shrinks to its speech plus trim_sec on each + // side (never beyond the cut itself), so the decoder gets little else. auto emit = [&](int64_t a, int64_t b, double end_sec) { - if (has_speech(a, std::min(b, n))) out.push_back({(double)a * fs, end_sec}); + const int64_t be = std::min(b, n); + if (!has_speech(a, be)) return; + double s0 = (double)a * fs, e0 = end_sec; + if (o.trim_sec > 0.0) { + int64_t fa = a, fb = be; + while (fa < fb && !sp[(size_t)fa]) ++fa; + while (fb > fa && !sp[(size_t)(fb - 1)]) --fb; + s0 = std::max(s0, (double)fa * fs - o.trim_sec); + e0 = std::min(e0, (double)fb * fs + o.trim_sec); + } + out.push_back({s0, e0}); }; int64_t s = 0; while (total_sec - (double)s * fs > o.max_seg_sec + 1e-9) { diff --git a/src/vad_segmenter.hpp b/src/vad_segmenter.hpp index 34b546e..a9842bf 100644 --- a/src/vad_segmenter.hpp +++ b/src/vad_segmenter.hpp @@ -18,6 +18,11 @@ struct SegmenterOpts { double bridge_sec = 0.1; // speech gaps shorter than this are bridged double min_speech_sec = 0.1; // speech runs shorter than this are dropped double pad_sec = 0.0; // speech_regions only: padding added on both sides of a region + // segment_by_vad only: a segment cut from long audio shrinks to its first + // and last speech frame plus this many seconds on each side. 0 keeps the + // whole cut (the behaviour before trimming existed). Audio of at most + // max_seg_sec is not affected. + double trim_sec = 0.3; }; // Which model made the probabilities. The two kinds differ in frame period @@ -53,11 +58,17 @@ SegmenterOpts default_segmenter_opts(VadKind kind); // 4. Segments that contain no speech are dropped, including the trailing // remainder. The result is empty when no segment has speech. Kept segments // are ordered and disjoint but need not touch. +// 5. With trim_sec > 0 each kept segment is cut down to [first speech frame - +// trim_sec, end of last speech frame + trim_sec], limited to the segment +// itself. Speech is the smoothed mask of step 1. This keeps the quiet or +// noisy parts that a hard or pause cut leaves at the edges away from the +// decoder. A segment is never trimmed to nothing: it has speech, so it keeps +// at least that run. Trimmed edges are not on the frame grid. // // Degenerate options (frame_sec not finite or <= 0, max_seg_sec not finite or -// <= 2 * frame_sec, threshold or any of the four durations not finite, any -// duration above 1e6 seconds) return the single segment {0, total_sec}. Every -// internal boundary is a whole number of frames. +// <= 2 * frame_sec, threshold, trim_sec or any of the four durations not finite, +// any duration above 1e6 seconds) return the single segment {0, total_sec}. +// Every cut between two segments is a whole number of frames. std::vector segment_by_vad(const std::vector& p, double total_sec, const SegmenterOpts& o); diff --git a/tests/test_transcribe_vad.cpp b/tests/test_transcribe_vad.cpp index c63e68d..d587613 100644 --- a/tests/test_transcribe_vad.cpp +++ b/tests/test_transcribe_vad.cpp @@ -72,6 +72,7 @@ int main() { { SegmenterOpts o; o.frame_sec = m->config().vad.frame_sec; + o.trim_sec = 0.0; // the cuts themselves are contiguous; trimming is tested in test_vad_trim_filter const auto segs = segment_by_vad(m->vad_probabilities(clip.samples), total, o); std::printf("62 s clip (%.2f s): %zu segments:", total, segs.size()); for (const auto& sg : segs) std::printf(" [%.2f-%.2f]", sg.start, sg.end); diff --git a/tests/test_vad_batched.cpp b/tests/test_vad_batched.cpp index 6e91b3d..02010d1 100644 --- a/tests/test_vad_batched.cpp +++ b/tests/test_vad_batched.cpp @@ -51,13 +51,16 @@ static void check_model(const char* path, const Audio& clip) { (double)cfg.hop_length * (double)cfg.subsampling_factor / (double)cfg.sample_rate; const double total = (double)clip.samples.size() / 16000.0; + // Trim 0.3 (the default) and 0 (the cuts as they were before trimming). + for (double trim : {0.3, 0.0}) for (double max_seg : {3.0, 7.0}) { SegmenterOpts opts; opts.max_seg_sec = max_seg; + opts.trim_sec = trim; SegmenterOpts so = opts; so.frame_sec = cfg.vad.frame_sec; const std::vector segs = segment_by_vad(m->vad_probabilities(clip.samples), total, so); - std::fprintf(stderr, " max_seg %.0f s: %zu segments\n", max_seg, segs.size()); + std::fprintf(stderr, " trim %.1f max_seg %.0f s: %zu segments\n", trim, max_seg, segs.size()); if (max_seg == 3.0) CHECK(segs.size() > 16); // more than one group of 16 CHECK(!segs.empty()); diff --git a/tests/test_vad_segmenter.cpp b/tests/test_vad_segmenter.cpp index 2eb4536..fb6e875 100644 --- a/tests/test_vad_segmenter.cpp +++ b/tests/test_vad_segmenter.cpp @@ -1,6 +1,7 @@ #include "vad_segmenter.hpp" #include +#include #include #include #include @@ -14,6 +15,12 @@ static int failures = 0; static const double FS = 0.08; +// Options with trimming off: the segmenter before trim_sec existed. +static SegmenterOpts legacy(SegmenterOpts o = SegmenterOpts()) { + o.trim_sec = 0.0; + return o; +} + // n frames of speech (p = 0.95) with silent ranges [a, b) in frames (p = 0.02). static std::vector make_p(int n, std::initializer_list> silences) { std::vector p((size_t)n, 0.95f); @@ -46,10 +53,13 @@ static void test_defaults() { CHECK(near(o.min_seg_sec, 1.0)); CHECK(near(o.bridge_sec, 0.1)); CHECK(near(o.min_speech_sec, 0.1)); + CHECK(near(o.trim_sec, 0.3)); + CHECK(near(default_segmenter_opts(VadKind::kHead).trim_sec, 0.3)); + CHECK(near(default_segmenter_opts(VadKind::kSilero).trim_sec, 0.3)); } static void test_short_is_single() { - SegmenterOpts o; + SegmenterOpts o = legacy(); auto s = segment_by_vad(make_p(250, {}), 20.0, o); CHECK(s.size() == 1 && s[0].start == 0.0 && s[0].end == 20.0); s = segment_by_vad({}, 0.5, o); // shorter than one frame @@ -62,7 +72,7 @@ static void test_short_is_single() { } static void test_threshold_is_inclusive() { - SegmenterOpts o; + SegmenterOpts o = legacy(); // 70 s of p == 0.5 exactly is all speech: hard cuts, nothing dropped. std::vector p(875, 0.5f); auto s = segment_by_vad(p, 70.0, o); @@ -73,7 +83,7 @@ static void test_threshold_is_inclusive() { } static void test_all_speech_hard_cuts() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 70.0; auto s = segment_by_vad(make_p((int)std::ceil(total / FS), {}), total, o); check_ordered(s, total, o.max_seg_sec); @@ -86,7 +96,7 @@ static void test_all_speech_hard_cuts() { } static void test_cuts_land_in_pauses() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 70.0; // pauses at [312,325) and [640,653): midpoints 318 (25.44 s) and 646 (51.68 s) auto p = make_p((int)std::ceil(total / FS), {{312, 325}, {640, 653}}); @@ -102,7 +112,7 @@ static void test_cuts_land_in_pauses() { } static void test_picks_last_pause_not_longest() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 40.0; // A long pause at 10-12 s and a short one (0.4 s) at 27 s. The last one wins. auto p = make_p((int)std::ceil(total / FS), {{125, 150}, {337, 342}}); @@ -113,7 +123,7 @@ static void test_picks_last_pause_not_longest() { } static void test_pause_must_be_fully_inside() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 45.0; // Pause 1 [200,230) is fully inside the window [0, 375]. Pause 2 [370,390) // starts inside but ends after 30 s, so it is not fully inside: pause 1 wins @@ -126,7 +136,7 @@ static void test_pause_must_be_fully_inside() { } static void test_midpoint_fallback() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 45.0; // Only pause: [350,390) = 28.0 to 31.2 s. It is not fully inside the 30 s // window, but its midpoint (370 = 29.6 s) is, so the cut goes there. @@ -143,7 +153,7 @@ static void test_midpoint_fallback() { } static void test_min_segment_one_second() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 31.0; // Pause [6,20): starts before 1 s, midpoint 13 = 1.04 s is inside: cut there. auto p = make_p((int)std::ceil(total / FS), {{6, 20}}); @@ -157,7 +167,7 @@ static void test_min_segment_one_second() { } static void test_min_pause() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 45.0; // 2 frames (0.16 s) is shorter than 0.2 s: hard cut at 30 s. auto s = segment_by_vad(make_p((int)std::ceil(total / FS), {{312, 314}}), total, o); @@ -168,7 +178,7 @@ static void test_min_pause() { } static void test_bridge_short_speech_gap() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 45.0; // A one-frame (0.08 s) dip inside speech is bridged, so it cannot form a // pause even when min_pause is set below one frame. @@ -183,7 +193,7 @@ static void test_bridge_short_speech_gap() { } static void test_drop_short_speech_runs() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 40.0; const int n = (int)std::ceil(total / FS); // Speech 0-5 s, then a lone one-frame blip at 34 s, silence elsewhere. The @@ -206,7 +216,7 @@ static void test_drop_short_speech_runs() { } static void test_segments_without_speech_are_dropped() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 70.0; const int n = (int)std::ceil(total / FS); // Speech for the first 10 s, then nothing. The pause [125,875) is not inside @@ -228,7 +238,7 @@ static void test_segments_without_speech_are_dropped() { } static void test_all_silence() { - SegmenterOpts o; + SegmenterOpts o = legacy(); const double total = 65.0; CHECK(segment_by_vad(make_p((int)std::ceil(total / FS), {{0, 100000}}), total, o).empty()); CHECK(segment_by_vad({}, 65.0, o).empty()); @@ -238,7 +248,7 @@ static void test_random_property() { std::mt19937 rng(7); std::uniform_real_distribution u(0.0f, 1.0f); std::uniform_int_distribution run_len(1, 12); - SegmenterOpts o; + SegmenterOpts o = legacy(); for (int trial = 0; trial < 200; ++trial) { const double total = 31.0 + 600.0 * u(rng); const int n = (int)std::ceil(total / FS); @@ -265,13 +275,13 @@ static void test_random_property() { static void test_degenerate_opts() { const double bad[] = {0.0, -0.08, INFINITY, NAN}; for (double fs : bad) { - SegmenterOpts o; + SegmenterOpts o = legacy(); o.frame_sec = fs; auto s = segment_by_vad(std::vector(100, 0.0f), 100.0, o); CHECK(s.size() == 1); if (s.size() == 1) CHECK(s[0].start == 0.0 && s[0].end == 100.0); } - SegmenterOpts o; + SegmenterOpts o = legacy(); o.max_seg_sec = 0.05; // smaller than a frame auto s = segment_by_vad(std::vector(100, 0.0f), 100.0, o); CHECK(s.size() == 1); @@ -285,7 +295,7 @@ static void test_nonfinite_and_huge_opts() { const double bads[] = {INFINITY, -INFINITY, NAN, 1e300, 1e7}; for (double v : bads) { for (int which = 0; which < 5; ++which) { - SegmenterOpts o; + SegmenterOpts o = legacy(); if (which == 0) o.min_seg_sec = v; else if (which == 1) o.min_pause_sec = v; else if (which == 2) o.max_seg_sec = v; @@ -298,7 +308,7 @@ static void test_nonfinite_and_huge_opts() { } const float thr[] = {INFINITY, -INFINITY, NAN}; for (float t : thr) { - SegmenterOpts o; + SegmenterOpts o = legacy(); o.threshold = t; auto s = segment_by_vad(std::vector(2000, 0.0f), 100.0, o); CHECK(s.size() == 1); @@ -309,7 +319,7 @@ static void test_nonfinite_and_huge_opts() { // speech_regions: smoothed speech runs for audio of any length, with gaps // shorter than min_pause merged. static void test_speech_regions() { - SegmenterOpts o; + SegmenterOpts o = legacy(); // 50 frames (4 s): silence 0-10, speech 10-20, silence 20-23 (0.24 s, a // pause), speech 23-35, silence 35-50. std::vector p(50, 0.02f); @@ -371,7 +381,7 @@ static void test_kind_defaults() { // Frames of 32 ms: boundaries are whole frames, a pause is found, caps hold. static void test_segmenter_32ms() { - SegmenterOpts o = default_segmenter_opts(VadKind::kSilero); + SegmenterOpts o = legacy(default_segmenter_opts(VadKind::kSilero)); const double fs = 0.032; // 50 s: speech, a 0.5 s pause at 20 s, speech to the end. const int n = (int)std::ceil(50.0 / fs); @@ -507,6 +517,179 @@ static void test_event_tracker_matches_offline() { CHECK(ev.size() == 2 && ev[0].start && near(ev[1].time, 0.5)); } + +// ---- trim_sec ------------------------------------------------------------ + +// n frames at p = 0.02 with speech (p = 0.95) in the given frame ranges. +static std::vector speech_p(int n, std::initializer_list> speech) { + std::vector p((size_t)n, 0.02f); + for (auto r : speech) + for (int i = r.first; i < r.second && i < n; ++i) p[(size_t)i] = 0.95f; + return p; +} + +static bool seg_is(const std::vector& s, std::initializer_list> want) { + if (s.size() != want.size()) return false; + size_t i = 0; + for (auto w : want) { + if (!near(s[i].start, w.first) || !near(s[i].end, w.second)) return false; + ++i; + } + return true; +} + +// Head frames (0.08 s), default trim 0.3 s. +static void test_trim_head() { + const SegmenterOpts o; // trim_sec 0.3 + // Speech [330,375) = 26.4-30.0 s and [450,500) = 36.0-40.0 s in 45 s. The + // cuts are at the middle of the leading silence (frame 165, which holds no + // speech, so that piece is dropped) and of the pause (frame 412 = 32.96 s). + auto p = speech_p(563, {{330, 375}, {450, 500}}); + auto s = segment_by_vad(p, 45.0, o); + CHECK(seg_is(s, {{26.4 - 0.3, 30.0 + 0.3}, {36.0 - 0.3, 40.0 + 0.3}})); + // trim 0 keeps the whole cuts. + s = segment_by_vad(p, 45.0, legacy()); + CHECK(seg_is(s, {{165 * FS, 412 * FS}, {412 * FS, 45.0}})); + // Speech at both edges of the audio: the start cannot go below 0 and the + // end not beyond the audio. Cut at the middle of the pause (frame 250). + p = speech_p(500, {{0, 10}, {490, 500}}); + s = segment_by_vad(p, 40.0, o); + CHECK(seg_is(s, {{0.0, 0.8 + 0.3}, {39.2 - 0.3, 40.0}})); + // No speech anywhere: nothing is left, and no segment is trimmed to nothing. + CHECK(segment_by_vad(speech_p(875, {}), 70.0, o).empty()); + // Two speech runs 0.16 s apart (not a pause) stay in one segment, and the + // gap between them is kept. The audio after the last run is dropped. + p = speech_p(563, {{100, 120}, {122, 140}}); + s = segment_by_vad(p, 45.0, o); + CHECK(seg_is(s, {{8.0 - 0.3, 11.2 + 0.3}})); + // Two runs far apart, in two segments: each is trimmed on its own, and the + // trim never reaches into the neighbouring segment. + p = speech_p(563, {{20, 40}, {60, 80}}); + s = segment_by_vad(p, 45.0, o); + CHECK(seg_is(s, {{1.6 - 0.3, 3.2 + 0.3}, {4.8 - 0.3, 6.4 + 0.3}})); + // A trim larger than the pause is limited by the cut (cut at frame 50 = 4.0 s). + SegmenterOpts big; + big.trim_sec = 5.0; + s = segment_by_vad(p, 45.0, big); + CHECK(s.size() == 2 && near(s[0].start, 0.0) && near(s[0].end, 4.0) && near(s[1].start, 4.0)); + check_ordered(s, 45.0, 30.0); +} + +// Silero frames (0.032 s). +static void test_trim_silero() { + const SegmenterOpts o = default_segmenter_opts(VadKind::kSilero); + CHECK(near(o.trim_sec, 0.3)); + const double fs = 0.032; + // Speech at the end of a window and a later run. 45 s, n = 1407. + // speech [900,937) and [1200,1250). The first cut is in the leading + // silence (frame 450), the second in the pause (frame 1068). + auto p = speech_p(1407, {{900, 937}, {1200, 1250}}); + auto s = segment_by_vad(p, 45.0, o); + CHECK(seg_is(s, {{900 * fs - 0.3, 937 * fs + 0.3}, {1200 * fs - 0.3, 1250 * fs + 0.3}})); + s = segment_by_vad(p, 45.0, legacy(o)); + CHECK(seg_is(s, {{450 * fs, 1068 * fs}, {1068 * fs, 45.0}})); + // Speech at both edges. 40 s, n = 1250, cut at frame 625 (20 s). + p = speech_p(1250, {{0, 40}, {1210, 1250}}); + s = segment_by_vad(p, 40.0, o); + CHECK(seg_is(s, {{0.0, 40 * fs + 0.3}, {1210 * fs - 0.3, 40.0}})); + // No speech. + CHECK(segment_by_vad(speech_p(1407, {}), 45.0, o).empty()); + // A short run (below min_speech 0.25 s) is no speech and is not kept. + CHECK(segment_by_vad(speech_p(1407, {{500, 507}}), 45.0, o).empty()); +} + +// Speech that runs through a hard cut is not trimmed at the cut, and a clip of +// at most max_seg_sec is returned whole. +static void test_trim_keeps_cuts_and_short_audio() { + const SegmenterOpts o; + auto s = segment_by_vad(speech_p(875, {{0, 875}}), 70.0, o); + CHECK(seg_is(s, {{0.0, 30.0}, {30.0, 60.0}, {60.0, 70.0}})); + s = segment_by_vad(speech_p(300, {{100, 110}}), 24.0, o); // 24 s: whole + CHECK(seg_is(s, {{0.0, 24.0}})); + s = segment_by_vad(speech_p(375, {{100, 110}}), 30.0, o); // exactly the cap + CHECK(seg_is(s, {{0.0, 30.0}})); + // Speech only in the leading 0.1 s of the second half of a hard cut. + s = segment_by_vad(speech_p(875, {{0, 375}, {375, 377}}), 70.0, o); + CHECK(s.size() == 2 && near(s[1].start, 30.0) && near(s[1].end, 30.0 + 0.16 + 0.3)); + // A bad trim is degenerate like the other options. + SegmenterOpts bad; + bad.trim_sec = NAN; + s = segment_by_vad(speech_p(875, {{0, 10}}), 70.0, bad); + CHECK(seg_is(s, {{0.0, 70.0}})); + // A negative trim means off. + bad.trim_sec = -1.0; + s = segment_by_vad(speech_p(563, {{20, 40}, {60, 80}}), 45.0, bad); + CHECK(s.size() == 2 && near(s[0].start, 0.0) && near(s[0].end, 4.0)); +} + +// Random streams: trimmed segments are inside the old cuts, contain the same +// speech, and never exceed trim_sec of non-speech at either edge. +static void test_trim_random_property() { + std::mt19937 rng(11); + for (int trial = 0; trial < 200; ++trial) { + const SegmenterOpts old = legacy(default_segmenter_opts(trial % 2 ? VadKind::kSilero : VadKind::kHead)); + SegmenterOpts nw = old; + nw.trim_sec = 0.1 * (double)(rng() % 10); + const double fs = old.frame_sec; + const int n = (int)(35.0 / fs) + (int)(rng() % 2000); + std::vector p; + bool sp = rng() & 1; + while ((int)p.size() < n) { + const int len = 1 + (int)(rng() % 300); + for (int i = 0; i < len && (int)p.size() < n; ++i) p.push_back(sp ? 0.9f : 0.05f); + sp = !sp; + } + const double total = (double)n * fs; + const auto a = segment_by_vad(p, total, old); + const auto b = segment_by_vad(p, total, nw); + CHECK(a.size() == b.size()); + for (size_t i = 0; i < a.size() && i < b.size(); ++i) { + CHECK(b[i].start >= a[i].start - 1e-9 && b[i].end <= a[i].end + 1e-9); + CHECK(b[i].end > b[i].start); + } + check_ordered(b, total, old.max_seg_sec); + } +} + +// With trim_sec 0 the output equals what the segmenter gave before trimming +// existed: the digest was computed with the previous implementation on the same +// seeded streams (80 ms and 32 ms frames, assorted options). +// Digest of segment_by_vad over seeded random inputs (FNV-1a over the printed segments). +static uint64_t digest_segments() { + std::mt19937 rng(20261004); + uint64_t h = 1469598103934665603ull; + auto mix = [&](const char* s) { for (; *s; ++s) { h ^= (unsigned char)*s; h *= 1099511628211ull; } }; + for (int kind = 0; kind < 2; ++kind) { + for (int iter = 0; iter < 150; ++iter) { + SegmenterOpts o = default_segmenter_opts(kind ? VadKind::kSilero : VadKind::kHead); + o.trim_sec = 0.0; + if (iter % 3 == 1) o.min_pause_sec = 0.05 + 0.01 * (double)(rng() % 40); + if (iter % 4 == 2) o.max_seg_sec = 5.0 + (double)(rng() % 250) / 10.0; + const int n = (int)(o.max_seg_sec / o.frame_sec) + 1 + (int)(rng() % 4000); + std::vector p; + bool sp = rng() & 1; + while ((int)p.size() < n) { + const int len = 1 + (int)(rng() % (iter % 2 ? 40 : 400)); + for (int i = 0; i < len && (int)p.size() < n; ++i) p.push_back(sp ? 0.9f : 0.05f); + sp = !sp; + } + const double total = (double)n * o.frame_sec - (double)(rng() % 100) / 100.0 * o.frame_sec * 0.9; + char buf[96]; + std::snprintf(buf, sizeof(buf), "#%d,%d;", kind, iter); + mix(buf); + for (const VadSegment& g : segment_by_vad(p, total, o)) { + std::snprintf(buf, sizeof(buf), "%.9f,%.9f;", g.start, g.end); + mix(buf); + } + } + } + return h; +} + +static void test_trim_zero_equals_previous_output() { + CHECK(digest_segments() == 16100934503807512709ull); +} + int main() { test_defaults(); test_nonfinite_and_huge_opts(); @@ -530,6 +713,11 @@ int main() { test_segmenter_32ms(); test_silero_speech_regions_and_pad(); test_event_tracker_matches_offline(); + test_trim_head(); + test_trim_silero(); + test_trim_keeps_cuts_and_short_audio(); + test_trim_random_property(); + test_trim_zero_equals_previous_output(); if (failures) return 1; std::puts("test_vad_segmenter: OK"); return 0; From 9e6ff57d2cc7648408f8deb0a5cab41caa7d0be5 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 4 Oct 2026 20:47:33 +0000 Subject: [PATCH 2/4] feat: add an opt-in word filter on per-word confidence An ASR model run on noise can emit words that were never said. Their per-word confidence is low, but so is the confidence of some real words, so a cut at one confidence value costs real words. The mean confidence of the neighbouring words separates the two much better: a doubtful word between confident ones stays, and a word that stands alone or among other doubtful words goes. Add pk::WordFilter and pk::apply_word_filter. A word is dropped when the mean confidence of the words that start within local_radius_sec (default 5 s) of it, the word included, is below min_local_conf. With drop_punct_only a word that is only punctuation is dropped too, for CTC models that emit a lone mark on noise. The filter works on one decode unit, removes the words from text, words and tokens, and leaves the transcription untouched when nothing is dropped or when it is off, which is the default. The count goes in Transcription::dropped_words (-1 when no filter ran). Words now record the index of their first and last token so the tokens can be removed with them. The filter is not wired to any entry point yet. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- src/transcription.cpp | 107 ++++++++++++++++++++ src/transcription.hpp | 34 +++++++ tests/CMakeLists.txt | 1 + tests/test_word_filter.cpp | 200 +++++++++++++++++++++++++++++++++++++ 4 files changed, 342 insertions(+) create mode 100644 tests/test_word_filter.cpp diff --git a/src/transcription.cpp b/src/transcription.cpp index c8623b5..fe33c53 100644 --- a/src/transcription.cpp +++ b/src/transcription.cpp @@ -3,6 +3,8 @@ #include #include +#include +#include #include #include #include @@ -184,6 +186,7 @@ std::vector group_words(const std::vector& tokens, w.start = (float)start[prev] * frame_sec; w.end = (float)end[built.back()] * frame_sec; w.conf = min_conf(built); + w.tok_first = built.front(); w.tok_last = built.back(); words.push_back(std::move(w)); } built.clear(); @@ -199,6 +202,7 @@ std::vector group_words(const std::vector& tokens, if (!lw.text.empty() && lw.text.back() == ' ') lw.text.pop_back(); lw.text += ct; lw.conf = std::min(lw.conf, conf[i]); + lw.tok_last = i; } else if (curr_punct && !built.empty()) { // Punctuation closing an open word: drop a trailing delimiter token, // then append this token. @@ -224,6 +228,7 @@ std::vector group_words(const std::vector& tokens, w.start = (float)start[prev] * frame_sec; w.end = (float)end[built.back()] * frame_sec; w.conf = min_conf(built); + w.tok_first = built.front(); w.tok_last = built.back(); words.push_back(std::move(w)); } } else if (!built.empty()) { @@ -232,10 +237,112 @@ std::vector group_words(const std::vector& tokens, w.start = (float)start[0] * frame_sec; w.end = (float)end[built.back()] * frame_sec; w.conf = min_conf(built); + w.tok_first = built.front(); w.tok_last = built.back(); words.push_back(std::move(w)); } return words; } +namespace { + +// True when the word has no letter or digit: only punctuation or spaces. +// ASCII is classified directly. Other code points count as content except the +// common punctuation blocks (General Punctuation, CJK symbols and punctuation, +// and the Latin-1 marks used in Spanish and French). +bool is_punct_only(const std::string& w) { + size_t i = 0; + while (i < w.size()) { + const unsigned char c = (unsigned char)w[i]; + uint32_t cp = c; + size_t len = 1; + if (c >= 0xF0 && i + 3 < w.size()) { cp = ((c & 0x07u) << 18) | (((unsigned char)w[i + 1] & 0x3Fu) << 12) | (((unsigned char)w[i + 2] & 0x3Fu) << 6) | ((unsigned char)w[i + 3] & 0x3Fu); len = 4; } + else if (c >= 0xE0 && i + 2 < w.size()) { cp = ((c & 0x0Fu) << 12) | (((unsigned char)w[i + 1] & 0x3Fu) << 6) | ((unsigned char)w[i + 2] & 0x3Fu); len = 3; } + else if (c >= 0xC0 && i + 1 < w.size()) { cp = ((c & 0x1Fu) << 6) | ((unsigned char)w[i + 1] & 0x3Fu); len = 2; } + i += len; + if (cp < 0x80) { + if (std::isalnum((int)cp)) return false; + continue; + } + const bool punct = (cp >= 0x2000 && cp <= 0x206F) || (cp >= 0x3000 && cp <= 0x303F) || + cp == 0xA1 || cp == 0xA7 || cp == 0xAB || cp == 0xB6 || cp == 0xB7 || + cp == 0xBB || cp == 0xBF; + if (!punct) return false; + } + return true; +} + +} // namespace + +int apply_word_filter(Transcription& t, const WordFilter& f) { + if (!f.active() || t.words.empty()) return 0; + const size_t n = t.words.size(); + std::vector drop(n, 0); + size_t dropped = 0; + if (f.min_local_conf > 0.0f) { + // Mean confidence of the words that start within the radius of a word, + // the word itself included. Words are in time order, so the window of + // each word is a range that moves forward; fall back to a scan when a + // caller passes words out of order. + bool sorted = true; + for (size_t i = 1; i < n && sorted; ++i) sorted = t.words[i - 1].start <= t.words[i].start; + std::vector cum(n + 1, 0.0); + for (size_t i = 0; i < n; ++i) cum[i + 1] = cum[i] + (double)t.words[i].conf; + const float r = f.local_radius_sec; + size_t lo = 0, hi = 0; // window [lo, hi) + for (size_t i = 0; i < n; ++i) { + const float s = t.words[i].start; + double sum = 0.0; + size_t cnt = 0; + if (sorted) { + while (lo < i && s - t.words[lo].start > r) ++lo; + if (hi < i + 1) hi = i + 1; + while (hi < n && t.words[hi].start - s <= r) ++hi; + sum = cum[hi] - cum[lo]; + cnt = hi - lo; + } else { + for (size_t j = 0; j < n; ++j) + if (std::fabs(t.words[j].start - s) <= r) { sum += (double)t.words[j].conf; ++cnt; } + } + if (cnt > 0 && sum / (double)cnt < (double)f.min_local_conf) drop[i] = 1; + } + } + if (f.drop_punct_only) + for (size_t i = 0; i < n; ++i) + if (is_punct_only(t.words[i].text)) drop[i] = 1; + for (size_t i = 0; i < n; ++i) dropped += drop[i] ? 1u : 0u; + t.dropped_words = (t.dropped_words < 0 ? 0 : t.dropped_words) + (int)dropped; + if (dropped == 0) return 0; + + // Rebuild words, text and tokens without the dropped words. + std::vector tdrop(t.tokens.size(), 0); + std::vector kept; + std::string text; + for (size_t i = 0; i < n; ++i) { + const Word& w = t.words[i]; + if (drop[i]) { + for (int32_t k = w.tok_first; k >= 0 && k <= w.tok_last && (size_t)k < tdrop.size(); ++k) tdrop[(size_t)k] = 1; + continue; + } + if (!text.empty()) text += ' '; + text += w.text; + kept.push_back(w); + } + std::vector remap(t.tokens.size(), -1); + std::vector toks; + for (size_t k = 0; k < t.tokens.size(); ++k) + if (!tdrop[k]) { remap[k] = (int32_t)toks.size(); toks.push_back(t.tokens[k]); } + for (Word& w : kept) { + int32_t a = -1, b = -1; + for (int32_t k = w.tok_first; k >= 0 && k <= w.tok_last && (size_t)k < remap.size(); ++k) + if (remap[(size_t)k] >= 0) { if (a < 0) a = remap[(size_t)k]; b = remap[(size_t)k]; } + w.tok_first = a; + w.tok_last = b; + } + t.words = std::move(kept); + t.tokens = std::move(toks); + t.text = std::move(text); + return (int)dropped; +} + } // namespace pk diff --git a/src/transcription.hpp b/src/transcription.hpp index 59d71ad..faddac4 100644 --- a/src/transcription.hpp +++ b/src/transcription.hpp @@ -19,6 +19,10 @@ struct Word { float start = 0.0f; float end = 0.0f; float conf = 0.0f; + // Indices of the word's first and last token in Transcription::tokens, or + // -1 when unknown. Used to remove a word's tokens with it. + int32_t tok_first = -1; + int32_t tok_last = -1; }; // A full transcription result: the flat text, the per-word timestamps + @@ -27,6 +31,9 @@ struct Transcription { std::string text; std::vector words; std::vector tokens; + // Number of words an optional word filter removed (see WordFilter). -1 when + // no filter ran, so a caller can tell "nothing dropped" from "no filter". + int dropped_words = -1; }; // Group a per-token decode (TokenInfo sequence, in emission order) into words, @@ -56,4 +63,31 @@ std::vector group_words(const std::vector& tokens, const std::vector& pieces, float frame_sec); +// Optional word filter, off by default. It removes words from a Transcription +// by their own confidence values, with no change to the decoding. +// +// min_local_conf : 0 = off. A word is dropped when the mean confidence of the +// words that start within local_radius_sec of its start (the +// word included) is below this value. A low confidence word +// between confident words keeps a high mean and stays. Words +// that stand alone, or only among other low confidence words, +// are dropped. This is what noise tends to produce. +// local_radius_sec: window for the mean, both sides, seconds (default 5). +// drop_punct_only: also drop words that are only punctuation (a CTC model +// can emit a lone "." or "?" on noise). +// +// A "decode unit" is one call of the decoder: the whole clip, or one VAD +// segment. Apply the filter to each unit alone, so that the mean never spans +// two segments. The dropped words leave `text`, `words` and `tokens`, and the +// token indices of the kept words are updated. When nothing is dropped the +// transcription is not changed. Returns the number of words dropped and adds it +// to `dropped_words` (which becomes 0 or more). +struct WordFilter { + float min_local_conf = 0.0f; + float local_radius_sec = 5.0f; + bool drop_punct_only = false; + bool active() const { return min_local_conf > 0.0f || drop_punct_only; } +}; +int apply_word_filter(Transcription& t, const WordFilter& f); + } // namespace pk diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3c9f63e..a15a216 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -18,6 +18,7 @@ pk_add_test(test_model_loader_ternary) pk_add_test(test_ternary_load_negative) pk_add_test(test_vad_head) pk_add_test(test_vad_segmenter) +pk_add_test(test_word_filter) pk_add_test(test_fft) pk_add_test(test_mel) pk_add_test(test_mel_gpu) diff --git a/tests/test_word_filter.cpp b/tests/test_word_filter.cpp new file mode 100644 index 0000000..fd1c513 --- /dev/null +++ b/tests/test_word_filter.cpp @@ -0,0 +1,200 @@ +// pk::apply_word_filter on scripted words and confidences (no model). +#include "transcription.hpp" + +#include +#include +#include +#include +#include + +using namespace pk; + +static int failures = 0; +#define CHECK(c) do { if (!(c)) { std::fprintf(stderr, "FAIL: %s (line %d)\n", #c, __LINE__); ++failures; } } while (0) + +struct W { const char* text; float start; float conf; }; + +// One token per word: token i has id i and belongs to word i. +static Transcription make(const std::vector& ws) { + Transcription t; + for (size_t i = 0; i < ws.size(); ++i) { + Word w; + w.text = ws[i].text; + w.start = ws[i].start; + w.end = w.start + 0.2f; + w.conf = ws[i].conf; + w.tok_first = w.tok_last = (int32_t)i; + t.words.push_back(w); + t.tokens.push_back(TokenInfo{(int32_t)i, (int32_t)(w.start / 0.08f), w.conf, 1}); + if (!t.text.empty()) t.text += ' '; + t.text += w.text; + } + return t; +} + +static bool same(const Transcription& a, const Transcription& b) { + if (a.text != b.text || a.words.size() != b.words.size() || a.tokens.size() != b.tokens.size() || + a.dropped_words != b.dropped_words) return false; + for (size_t i = 0; i < a.words.size(); ++i) { + const Word &x = a.words[i], &y = b.words[i]; + if (x.text != y.text || x.start != y.start || x.end != y.end || x.conf != y.conf || + x.tok_first != y.tok_first || x.tok_last != y.tok_last) return false; + } + for (size_t i = 0; i < a.tokens.size(); ++i) + if (a.tokens[i].id != b.tokens[i].id || a.tokens[i].frame != b.tokens[i].frame || + a.tokens[i].conf != b.tokens[i].conf || a.tokens[i].span != b.tokens[i].span) return false; + return true; +} + +static void test_off_is_identity() { + const Transcription ref = make({{"a", 0.f, 0.9f}, {"b", 1.f, 0.05f}, {".", 2.f, 0.1f}}); + Transcription t = ref; + CHECK(!WordFilter().active()); + CHECK(WordFilter().min_local_conf == 0.0f && WordFilter().local_radius_sec == 5.0f && !WordFilter().drop_punct_only); + CHECK(apply_word_filter(t, WordFilter()) == 0); + CHECK(same(t, ref) && t.dropped_words == -1); + // A radius alone does not turn the filter on. + WordFilter f; + f.local_radius_sec = 1.0f; + CHECK(!f.active() && apply_word_filter(t, f) == 0 && same(t, ref)); + // Empty input. + Transcription e; + f.min_local_conf = 0.5f; + CHECK(apply_word_filter(e, f) == 0 && e.words.empty() && e.dropped_words == -1); +} + +static void test_isolated_low_word_between_confident_words_is_kept() { + Transcription t = make({{"a", 0.0f, 1.0f}, {"b", 0.3f, 0.2f}, {"c", 0.6f, 1.0f}}); + WordFilter f; + f.min_local_conf = 0.5f; + CHECK(apply_word_filter(t, f) == 0); + CHECK(t.words.size() == 3 && t.text == "a b c" && t.tokens.size() == 3); + CHECK(t.dropped_words == 0); // the filter ran and dropped nothing +} + +static void test_run_of_low_words_is_dropped() { + // Speech, then a run of low confidence words 20 s later, then speech. + Transcription t = make({{"hello", 0.0f, 0.95f}, {"world", 0.5f, 0.9f}, + {"x", 20.0f, 0.2f}, {"y", 20.4f, 0.3f}, {"z", 20.8f, 0.25f}, + {"again", 40.0f, 0.9f}}); + WordFilter f; + f.min_local_conf = 0.5f; + CHECK(apply_word_filter(t, f) == 3); + CHECK(t.dropped_words == 3); + CHECK(t.text == "hello world again"); + CHECK(t.words.size() == 3 && t.tokens.size() == 3); + CHECK(t.tokens[0].id == 0 && t.tokens[1].id == 1 && t.tokens[2].id == 5); + CHECK(t.words[2].text == "again" && t.words[2].tok_first == 2 && t.words[2].tok_last == 2); + // A lone low confidence word with no neighbour in the radius is dropped too. + t = make({{"a", 0.0f, 0.9f}, {"lone", 30.0f, 0.4f}}); + CHECK(apply_word_filter(t, f) == 1 && t.text == "a"); +} + +static void test_radius_and_boundaries() { + WordFilter f; + f.min_local_conf = 0.5f; + f.local_radius_sec = 5.0f; + // B is exactly 5 s after A: inside the window (the bound is inclusive), so + // B's mean is 0.6 and it stays. + Transcription t = make({{"A", 0.0f, 1.0f}, {"B", 5.0f, 0.2f}}); + CHECK(apply_word_filter(t, f) == 0); + // Just under 5 s of radius: B is alone, mean 0.2, dropped. A keeps its own 1.0. + f.local_radius_sec = 4.99f; + t = make({{"A", 0.0f, 1.0f}, {"B", 5.0f, 0.2f}}); + CHECK(apply_word_filter(t, f) == 1 && t.text == "A"); + // The window looks both ways: a low word followed by confident words stays. + f.local_radius_sec = 5.0f; + t = make({{"B", 0.0f, 0.2f}, {"A", 1.0f, 1.0f}, {"C", 2.0f, 1.0f}}); + CHECK(apply_word_filter(t, f) == 0); + // A threshold equal to the mean keeps the words (strictly below is dropped). + f.min_local_conf = 0.75f; + t = make({{"A", 0.0f, 1.0f}, {"B", 1.0f, 0.5f}}); // both have mean 0.75 + CHECK(apply_word_filter(t, f) == 0); + f.min_local_conf = 0.76f; + t = make({{"A", 0.0f, 1.0f}, {"B", 1.0f, 0.5f}}); + CHECK(apply_word_filter(t, f) == 2 && t.words.empty() && t.text.empty() && t.tokens.empty()); + // Words out of time order give the same answer as sorted ones. + f.min_local_conf = 0.5f; + f.local_radius_sec = 2.0f; + t = make({{"x", 20.0f, 0.2f}, {"a", 0.0f, 0.9f}, {"y", 20.5f, 0.3f}, {"b", 0.5f, 0.9f}}); + CHECK(apply_word_filter(t, f) == 2 && t.text == "a b"); +} + +static void test_punct_only() { + WordFilter f; + f.drop_punct_only = true; + CHECK(f.active()); + Transcription t = make({{"a.", 0.0f, 0.9f}, {".", 1.0f, 0.9f}, {"?!", 2.0f, 0.9f}, {"...", 3.0f, 0.9f}, + {"\xE2\x80\xA6", 4.0f, 0.9f}, // ellipsis + {"\xE2\x80\x94", 5.0f, 0.9f}, // em dash + {"\xC2\xBF", 6.0f, 0.9f}, // inverted question mark + {"1", 7.0f, 0.9f}, {"\xC3\xA9", 8.0f, 0.9f}, // e acute is a letter + {"\xE6\x97\xA5", 9.0f, 0.9f}, // a CJK letter + {"-5", 10.0f, 0.9f}}); + CHECK(apply_word_filter(t, f) == 6); + CHECK(t.words.size() == 5); + CHECK(t.text == "a. 1 \xC3\xA9 \xE6\x97\xA5 -5"); + CHECK(t.tokens.size() == 5 && t.tokens[1].id == 7 && t.words[1].tok_first == 1); + // The local filter does not touch confident punctuation, and punct-only + // words count in the mean (that is the input the decoder gave). + f.drop_punct_only = false; + f.min_local_conf = 0.5f; + t = make({{".", 0.0f, 0.9f}, {"a", 1.0f, 0.9f}}); + CHECK(apply_word_filter(t, f) == 0); +} + +static void test_both_checks_together_and_accumulate() { + WordFilter f; + f.min_local_conf = 0.5f; + f.drop_punct_only = true; + Transcription t = make({{"a", 0.0f, 0.9f}, {".", 0.5f, 0.9f}, {"n", 30.0f, 0.1f}}); + CHECK(apply_word_filter(t, f) == 2 && t.text == "a" && t.dropped_words == 2); + // A second call on the same transcription adds to the count. + Word q; + q.text = "q"; q.start = 30.0f; q.end = 30.2f; q.conf = 0.1f; q.tok_first = q.tok_last = (int32_t)t.tokens.size(); + t.words.push_back(q); + t.tokens.push_back(TokenInfo{9, 12, 0.1f, 1}); + t.text += " q"; + CHECK(apply_word_filter(t, f) == 1 && t.dropped_words == 3 && t.text == "a"); +} + +static void test_group_words_records_token_ranges() { + // "hello", "world," (two pieces and a comma), "hi". + const std::vector pieces = {"\xE2\x96\x81hello", "\xE2\x96\x81wor", "ld", ",", "\xE2\x96\x81hi"}; + std::vector tokens; + for (int i = 0; i < 5; ++i) tokens.push_back(TokenInfo{i, i * 2, 0.9f, 1}); + const std::vector ws = group_words(tokens, pieces, 0.08f); + CHECK(ws.size() == 3); + if (ws.size() == 3) { + CHECK(ws[0].text == "hello" && ws[0].tok_first == 0 && ws[0].tok_last == 0); + CHECK(ws[1].text == "world," && ws[1].tok_first == 1 && ws[1].tok_last == 3); + CHECK(ws[2].text == "hi" && ws[2].tok_first == 4 && ws[2].tok_last == 4); + } + // Dropping the middle word removes its three tokens and renumbers the last. + Transcription t; + t.words = ws; + t.tokens = tokens; + t.words[1].conf = 0.05f; + t.words[0].start = 0.0f; t.words[1].start = 30.0f; t.words[2].start = 60.0f; + for (Word& w : t.words) { w.conf = 0.9f; } + t.words[1].conf = 0.05f; + t.text = "hello world, hi"; + WordFilter f; + f.min_local_conf = 0.5f; + CHECK(apply_word_filter(t, f) == 1); + CHECK(t.text == "hello hi" && t.tokens.size() == 2 && t.tokens[0].id == 0 && t.tokens[1].id == 4); + CHECK(t.words[1].tok_first == 1 && t.words[1].tok_last == 1); +} + +int main() { + test_off_is_identity(); + test_isolated_low_word_between_confident_words_is_kept(); + test_run_of_low_words_is_dropped(); + test_radius_and_boundaries(); + test_punct_only(); + test_both_checks_together_and_accumulate(); + test_group_words_records_token_ranges(); + if (failures) return 1; + std::puts("test_word_filter: OK"); + return 0; +} From 224236b63c64b0b308355c7337aa6f6b63e18715 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 4 Oct 2026 20:49:28 +0000 Subject: [PATCH 3/4] feat: expose the segment trim and the word filter in the C-API and the CLI Segment trim: a "trim" key (seconds >= 0, 0 = off, default 0.3) in the VAD options JSON, so it applies to the "segments" mode of the VAD functions and to parakeet_capi_transcribe_path_json_vad_with. CLI: --vad-trim on transcribe and --trim on the vad command. Word filter, off by default: - Model::transcribe_pcm_vad and transcribe_pcm_vad_with_timestamps take an optional WordFilter that runs on each decode unit, the whole clip or one VAD segment, before the segment offsets are added. - New parakeet_capi_transcribe_path_json_with(ctx, wav, decoder, options) for a plain transcribe with the filter keys min_local_conf, local_radius and drop_punct_only. Additive, ABI stays 10. The same keys work in parakeet_capi_transcribe_path_json_vad_with. - CLI: --min-local-conf, --local-radius, --drop-punct-only, with or without --vad. - When a filter ran, the JSON document has "guard":{"dropped_words":N}. Without a filter the document is the same as before, byte for byte. Tests: option parsing and errors (test_vad_options, ctest cases for the CLI flags), and model tests (test_vad_trim_filter) for the trimmed offsets, trim 0 against the old cuts, the filter off being identical, and the filter on, with the VAD head, Silero and a CTC model. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- examples/cli/main.cpp | 72 ++++++- include/parakeet_capi.h | 38 +++- src/model.cpp | 19 +- src/model.hpp | 12 +- src/parakeet_capi.cpp | 41 +++- src/transcription_json.cpp | 9 +- src/transcription_json.hpp | 3 +- src/vad_json.cpp | 84 ++++++--- src/vad_json.hpp | 21 ++- tests/CMakeLists.txt | 17 ++ tests/test_vad_options.cpp | 54 ++++++ tests/test_vad_trim_filter.cpp | 331 +++++++++++++++++++++++++++++++++ 12 files changed, 649 insertions(+), 52 deletions(-) create mode 100644 tests/test_vad_trim_filter.cpp diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 7ebc497..c60a46a 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -373,8 +373,9 @@ static int cmd_transcribe_stream(const std::string& model, const std::string& in // Segmenter options the user set on the command line. A value left unset keeps // the default of the VAD in use (Ultra/Redux head or Silero). struct VadOverrides { - std::optional threshold, min_pause, min_speech, max_seg, pad; + std::optional threshold, min_pause, min_speech, max_seg, pad, trim; void apply(pk::SegmenterOpts& o) const { + if (trim) o.trim_sec = *trim; if (threshold) o.threshold = (float)*threshold; if (min_pause) o.min_pause_sec = *min_pause; if (min_speech) o.min_speech_sec = *min_speech; @@ -386,7 +387,7 @@ struct VadOverrides { static int cmd_transcribe_vad(const std::string& model, const std::string& input, pk::Decoder dec, const std::string& lang, bool timestamps, bool json, const VadOverrides& ov, const std::string& vad_model, - const std::string& vad_component) { + const std::string& vad_component, const pk::WordFilter& wf) { pk::Audio audio; if (!load_audio_arg_16k_mono(input, audio)) { std::fprintf(stderr, "parakeet-cli: failed to load audio %s\n", input.c_str()); @@ -433,7 +434,7 @@ static int cmd_transcribe_vad(const std::string& model, const std::string& input const pk::Model::VadProbabilityFn* ext = silero ? &fn : nullptr; if (json || timestamps) { pk::Transcription tr = - m->transcribe_pcm_vad_with_timestamps(audio.samples, audio.sample_rate, dec, lang, opts, ext); + m->transcribe_pcm_vad_with_timestamps(audio.samples, audio.sample_rate, dec, lang, opts, ext, wf); if (json) { std::printf("%s\n", pk::transcription_to_json(tr, model_frame_sec(*m)).c_str()); } else { @@ -441,7 +442,7 @@ static int cmd_transcribe_vad(const std::string& model, const std::string& input std::printf("%.2f-%.2f %s (%.2f)\n", w.start, w.end, w.text.c_str(), w.conf); } } else { - std::printf("%s\n", m->transcribe_pcm_vad(audio.samples, audio.sample_rate, dec, lang, opts, ext).c_str()); + std::printf("%s\n", m->transcribe_pcm_vad(audio.samples, audio.sample_rate, dec, lang, opts, ext, wf).c_str()); } } catch (const std::exception& e) { std::fprintf(stderr, "parakeet-cli: transcribe failed: %s\n", e.what()); @@ -463,12 +464,18 @@ static int cmd_transcribe(int argc, char** argv) { bool vad = false; std::string vad_model, vad_component, component; VadOverrides vad_ov; + pk::WordFilter word_filter; double d = 0.0; auto parse_pos = [](const char* str, double& out) { char* end = nullptr; out = std::strtod(str, &end); return end != str && *end == '\0' && std::isfinite(out) && out > 0.0 && out <= 1e6; }; + auto parse_nonneg = [](const char* str, double& out) { + char* end = nullptr; + out = std::strtod(str, &end); + return end != str && *end == '\0' && std::isfinite(out) && out >= 0.0 && out <= 1e6; + }; bool score_norm = true; int beam_size = 0; int nbest = 0; @@ -515,6 +522,17 @@ static int cmd_transcribe(int argc, char** argv) { } else if (std::strcmp(argv[i], "--vad-min-speech") == 0 && i + 1 < argc) { if (!parse_pos(argv[++i], d)) { std::fprintf(stderr, "parakeet-cli: --vad-min-speech must be > 0\n"); return 2; } vad_ov.min_speech = d; + } else if (std::strcmp(argv[i], "--vad-trim") == 0 && i + 1 < argc) { + if (!parse_nonneg(argv[++i], d)) { std::fprintf(stderr, "parakeet-cli: --vad-trim must be >= 0 (0 = keep the whole cuts)\n"); return 2; } + vad_ov.trim = d; + } else if (std::strcmp(argv[i], "--min-local-conf") == 0 && i + 1 < argc) { + if (!parse_nonneg(argv[++i], d) || d > 1.0) { std::fprintf(stderr, "parakeet-cli: --min-local-conf must be in [0,1] (0 = off)\n"); return 2; } + word_filter.min_local_conf = (float)d; + } else if (std::strcmp(argv[i], "--local-radius") == 0 && i + 1 < argc) { + if (!parse_pos(argv[++i], d)) { std::fprintf(stderr, "parakeet-cli: --local-radius must be > 0\n"); return 2; } + word_filter.local_radius_sec = (float)d; + } else if (std::strcmp(argv[i], "--drop-punct-only") == 0) { + word_filter.drop_punct_only = true; } else if (std::strcmp(argv[i], "--vad-max-seg") == 0 && i + 1 < argc) { if (!parse_pos(argv[++i], d)) { std::fprintf(stderr, "parakeet-cli: --vad-max-seg must be > 0\n"); return 2; } vad_ov.max_seg = d; @@ -527,7 +545,8 @@ static int cmd_transcribe(int argc, char** argv) { "[--threads N] [--json] " "[--component NAME] " "[--vad [--vad-model ] [--vad-component NAME] [--vad-threshold F=0.5] [--vad-min-pause SEC] " - "[--vad-min-speech SEC] [--vad-max-seg SEC=30]] " + "[--vad-min-speech SEC] [--vad-max-seg SEC=30] [--vad-trim SEC=0.3]] " + "[--min-local-conf F [--local-radius SEC=5]] [--drop-punct-only] " "[--beam-size N [--nbest N] [--no-score-norm]]\n"); return 2; } @@ -538,6 +557,10 @@ static int cmd_transcribe(int argc, char** argv) { if (threads > 0) pk::set_num_threads(threads); if (stream) { + if (word_filter.active()) { + std::fprintf(stderr, "parakeet-cli: the word filter is offline only\n"); + return 2; + } if (vad) { std::fprintf(stderr, "parakeet-cli: --vad is offline only\n"); return 2; @@ -581,7 +604,39 @@ static int cmd_transcribe(int argc, char** argv) { std::fprintf(stderr, "parakeet-cli: --vad works with greedy decoding only\n"); return 2; } - return cmd_transcribe_vad(model, input, dec, lang, timestamps, json, vad_ov, vad_model, vad_component); + return cmd_transcribe_vad(model, input, dec, lang, timestamps, json, vad_ov, vad_model, vad_component, word_filter); + } + if (word_filter.active() && (beam_size != 0 || nbest != 0)) { + std::fprintf(stderr, "parakeet-cli: the word filter works with greedy decoding only\n"); + return 2; + } + if (word_filter.active()) { + // Greedy decode with timestamps, then the word filter on the whole clip. + pk::Audio audio; + if (is_stdin_input(input) && !load_audio_arg_16k_mono(input, audio)) { + std::fprintf(stderr, "parakeet-cli: failed to load audio stdin\n"); + return 1; + } + try { + std::unique_ptr m = load_asr(model); + if (!m) { std::fprintf(stderr, "parakeet-cli: failed to load model %s\n", model.c_str()); return 1; } + pk::Transcription tr = is_stdin_input(input) + ? m->transcribe_with_timestamps(audio.samples, audio.sample_rate, dec, lang) + : m->transcribe_path_with_timestamps(input, dec, lang); + pk::apply_word_filter(tr, word_filter); + if (json) { + std::printf("%s\n", pk::transcription_to_json(tr, model_frame_sec(*m)).c_str()); + } else if (timestamps) { + for (const pk::Word& w : tr.words) + std::printf("%.2f-%.2f %s (%.2f)\n", w.start, w.end, w.text.c_str(), w.conf); + } else { + std::printf("%s\n", tr.text.c_str()); + } + } catch (const std::exception& e) { + std::fprintf(stderr, "parakeet-cli: transcribe failed: %s\n", e.what()); + return 1; + } + return 0; } if (nbest != 0 && beam_size == 0) { std::fprintf(stderr, @@ -2245,6 +2300,9 @@ static int cmd_vad(int argc, char** argv) { } else if (std::strcmp(argv[i], "--max-segment") == 0 && i + 1 < argc) { if (!num(argv[++i], d, false)) return bad("--max-segment must be > 0"); ov.max_seg = d; + } else if (std::strcmp(argv[i], "--trim") == 0 && i + 1 < argc) { + if (!num(argv[++i], d, true)) return bad("--trim must be >= 0"); + ov.trim = d; } else if (std::strcmp(argv[i], "--mode") == 0 && i + 1 < argc) { const char* v = argv[++i]; if (std::strcmp(v, "speech") == 0) mode = pk::VadRequest::Mode::kSpeech; @@ -2258,7 +2316,7 @@ static int cmd_vad(int argc, char** argv) { std::fprintf(stderr, "usage: parakeet-cli vad --model --input " "[--component NAME] [--threshold F=0.5] [--min-pause SEC] [--min-speech SEC] [--speech-pad SEC] " - "[--max-segment SEC=30] [--mode speech|segments] [--probabilities] [--threads N]\n"); + "[--max-segment SEC=30] [--trim SEC=0.3] [--mode speech|segments] [--probabilities] [--threads N]\n"); return 2; } if (threads > 0) pk::set_num_threads(threads); diff --git a/include/parakeet_capi.h b/include/parakeet_capi.h index 4640853..69ebe9d 100644 --- a/include/parakeet_capi.h +++ b/include/parakeet_capi.h @@ -74,6 +74,8 @@ typedef struct parakeet_ctx parakeet_ctx; // parakeet_capi_transcribe_path_json_vad*) and Silero VAD contexts are // additive and keep ABI v10: a caller that needs them checks for the symbols // (dlsym) or for PARAKEET_MODEL_KIND_VAD. +// Word filter (parakeet_capi_transcribe_path_json_with and the filter keys of +// parakeet_capi_transcribe_path_json_vad_with) is additive and keeps ABI v10. // Bundle GGUF (parakeet_capi_load_component, parakeet_capi_bundle_components_json, // parakeet_capi_load_error; docs/bundle.md) is additive and keeps ABI v10. int parakeet_capi_abi_version(void); @@ -199,13 +201,37 @@ int parakeet_capi_transcribe_pcm_batch_lang(parakeet_ctx* ctx, char* parakeet_capi_transcribe_path_json(parakeet_ctx* ctx, const char* wav_path, int decoder); +// Like parakeet_capi_transcribe_path_json, with the optional word filter +// (additive; no ABI bump). `options_json` is a flat JSON object, or NULL / "" for +// no filter, in which case the document is the one of the plain function. +// Keys: +// "min_local_conf" 0 to 1; 0 = off (default). A word is dropped when the mean +// confidence of the words that start within "local_radius" +// seconds of it (itself included) is below this value. It +// removes words that stand alone or sit among other low +// confidence words, as noise tends to give, and keeps a +// doubtful word between confident ones. 0.5 is a good +// start. Higher values also drop real words on some models +// (see docs/vad-benchmarks.md). +// "local_radius" seconds > 0; default 5. +// "drop_punct_only" true to drop words that are only punctuation (a CTC model +// can emit a lone "." on noise); default false. +// Unknown keys and bad values are errors (NULL, last error set). The filter only +// post-processes the decode: with it off, or when it drops nothing, the words and +// tokens are the same as without it. When a filter is on, the document gets one +// more member, "guard":{"dropped_words":N}, with N = 0 when nothing was dropped; +// the dropped words are also removed from "text", "words" and "tokens". The +// filter works on the whole file as one decode unit. +char* parakeet_capi_transcribe_path_json_with(parakeet_ctx* ctx, const char* wav_path, + int decoder, const char* options_json); + // Like parakeet_capi_transcribe_path_json, but long audio is cut at pauses found // by the model's own VAD head into segments of at most 30 s, and the segments are // transcribed one by one (word/token times are relative to the whole file). Audio // of 30 s or less gives the same document as the plain function. Returns NULL and // sets the context's last error to "model has no VAD head" when the model has no // VAD head. It always uses the default segmenter options (30 s cap, threshold -// 0.5). Additive; no ABI bump. +// 0.5, each segment trimmed to its speech plus 0.3 s). Additive; no ABI bump. char* parakeet_capi_transcribe_path_json_vad(parakeet_ctx* ctx, const char* wav_path, int decoder); @@ -237,6 +263,9 @@ char* parakeet_capi_transcribe_path_json_vad(parakeet_ctx* ctx, const char* wav_ // "speech_pad" seconds >= 0; "speech" mode: each region is widened by this // on both sides; default 0 (Silero: 0.03) // "max_segment" seconds; segment cap in "segments" mode; default 30 +// "trim" seconds >= 0; "segments" mode: each segment shrinks to its +// first and last speech frame plus this much; default 0.3; +// 0 keeps the whole cuts // "mode" "speech" (default) or "segments" // "probabilities" true to add the per-frame probabilities; default false // Unknown keys and out-of-range values are errors. The Silero defaults are the @@ -277,8 +306,11 @@ char* parakeet_capi_vad_path_json(parakeet_ctx* ctx, const char* wav_path, // Silero context, or NULL to use the ASR model's own head (then the result is // as parakeet_capi_transcribe_path_json_vad, with the options below). Options // are the JSON object of parakeet_capi_vad_pcm_json; only "threshold", -// "min_pause", "min_speech" and "max_segment" are used here (the other keys are -// accepted and ignored). NULL or "" gives the defaults of the VAD in use. +// "min_pause", "min_speech", "max_segment" and "trim" are used here (the other +// keys of that object are accepted and ignored), plus the word filter keys of +// parakeet_capi_transcribe_path_json_with, which apply to each segment on its +// own (and to the whole file when it is at most max_segment seconds). NULL or +// "" gives the defaults of the VAD in use. // Audio of at most max_segment seconds (30 by default) is transcribed whole, // without running the VAD. Word and token times are relative to the whole file. // Returns the same document as parakeet_capi_transcribe_path_json. Errors set diff --git a/src/model.cpp b/src/model.cpp index 4c0bb42..d1d25a5 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -462,7 +462,10 @@ std::vector vad_slices(const Model& m, const std::vector& pcm16k, std::string Model::transcribe_pcm_vad(const std::vector& pcm, int sample_rate, Decoder decoder, const std::string& target_lang, const SegmenterOpts& opts, - const VadProbabilityFn* external_vad) const { + const VadProbabilityFn* external_vad, + const WordFilter& filter) const { + if (filter.active()) + return transcribe_pcm_vad_with_timestamps(pcm, sample_rate, decoder, target_lang, opts, external_vad, filter).text; PoolLease lease(pool_snapshot()); if (!external_vad && !loader_.config().vad.present) throw std::runtime_error("model has no VAD head"); const std::vector pcm16k = @@ -486,22 +489,28 @@ std::string Model::transcribe_pcm_vad(const std::vector& pcm, int sample_ Transcription Model::transcribe_pcm_vad_with_timestamps(const std::vector& pcm, int sample_rate, Decoder decoder, const std::string& target_lang, const SegmenterOpts& opts, - const VadProbabilityFn* external_vad) const { + const VadProbabilityFn* external_vad, + const WordFilter& filter) const { PoolLease lease(pool_snapshot()); if (!external_vad && !loader_.config().vad.present) throw std::runtime_error("model has no VAD head"); const std::vector pcm16k = sample_rate == 16000 ? pcm : resample_linear(pcm, sample_rate, 16000); - if ((double)pcm16k.size() / 16000.0 <= opts.max_seg_sec) - return transcribe_with_timestamps(pcm16k, 16000, decoder, target_lang); + if ((double)pcm16k.size() / 16000.0 <= opts.max_seg_sec) { + Transcription t = transcribe_with_timestamps(pcm16k, 16000, decoder, target_lang); + apply_word_filter(t, filter); + return t; + } const std::vector slices = vad_slices(*this, pcm16k, opts, external_vad); - if (slices.empty()) return Transcription(); // no speech found Transcription all; + if (filter.active()) all.dropped_words = 0; + if (slices.empty()) return all; // no speech found std::vector*> pcms; for (const Slice& s : slices) pcms.push_back(&s.pcm); std::vector parts = transcribe_16k_grouped(pcms, decoder, target_lang, true); for (size_t i = 0; i < slices.size(); ++i) { const Slice& s = slices[i]; Transcription& t = parts[i]; + if (filter.active()) all.dropped_words += apply_word_filter(t, filter); for (Word& w : t.words) { w.start += (float)s.start_sec; w.end += (float)s.start_sec; } for (TokenInfo& k : t.tokens) k.frame += s.start_frame; if (!t.text.empty()) { diff --git a/src/model.hpp b/src/model.hpp index 9945842..6a5317b 100644 --- a/src/model.hpp +++ b/src/model.hpp @@ -162,17 +162,25 @@ class Model { // PCM to one speech probability per opts.frame_sec seconds, and opts.frame_sec // must then be set to that period. The segmenter, the 30 s cap and the decode // of each segment are the same as with the head. + // + // `filter` (optional, off by default, see WordFilter) removes words by their + // local confidence. It runs on each decode unit alone: the whole clip when it + // is at most opts.max_seg_sec, else each VAD segment. With a filter on, the + // text path also runs the timestamp decode, and the Transcription reports + // the words dropped in `dropped_words`. using VadProbabilityFn = std::function(const std::vector&)>; std::string transcribe_pcm_vad(const std::vector& pcm, int sample_rate, Decoder decoder = Decoder::kDefault, const std::string& target_lang = "", const SegmenterOpts& opts = SegmenterOpts(), - const VadProbabilityFn* external_vad = nullptr) const; + const VadProbabilityFn* external_vad = nullptr, + const WordFilter& filter = WordFilter()) const; Transcription transcribe_pcm_vad_with_timestamps( const std::vector& pcm, int sample_rate, Decoder decoder = Decoder::kDefault, const std::string& target_lang = "", const SegmenterOpts& opts = SegmenterOpts(), - const VadProbabilityFn* external_vad = nullptr) const; + const VadProbabilityFn* external_vad = nullptr, + const WordFilter& filter = WordFilter()) const; // The underlying loaded GGUF. Exposed so the streaming C-API can build a // pk::StreamingSession (and a MelFrontend) over the same load-once model. diff --git a/src/parakeet_capi.cpp b/src/parakeet_capi.cpp index b3d5e5c..0d73fc9 100644 --- a/src/parakeet_capi.cpp +++ b/src/parakeet_capi.cpp @@ -651,6 +651,43 @@ extern "C" char* parakeet_capi_transcribe_path_json(parakeet_ctx* ctx, } } +extern "C" char* parakeet_capi_transcribe_path_json_with(parakeet_ctx* ctx, const char* wav_path, + int decoder, const char* options_json) { + if (!ctx) return nullptr; + if (!ctx->model) { + ctx->last_error = ctx->diar + ? "context holds a diarization model; use parakeet_capi_diarize_*" + : no_model_msg(ctx); + return nullptr; + } + if (!wav_path) { ctx->last_error = "wav_path is NULL"; return nullptr; } + try { + pk::WordFilter filter; + std::string err; + if (!pk::parse_filter_options(options_json, filter, err)) { + ctx->last_error = err; + return nullptr; + } + pk::Transcription tr = + ctx->model->transcribe_path_with_timestamps(wav_path, to_decoder(decoder)); + pk::apply_word_filter(tr, filter); + const pk::ParakeetConfig& cfg = ctx->model->config(); + const float frame_sec = + (float)cfg.hop_length * (float)cfg.subsampling_factor / (float)cfg.sample_rate; + std::string json = pk::transcription_to_json(tr, frame_sec); + ctx->last_error.clear(); + char* out = dup_to_c(json); + if (!out) { ctx->last_error = "out of memory"; return nullptr; } + return out; + } catch (const std::exception& e) { + ctx->last_error = e.what(); + return nullptr; + } catch (...) { + ctx->last_error = "unknown error"; + return nullptr; + } +} + extern "C" char* parakeet_capi_transcribe_path_json_vad(parakeet_ctx* ctx, const char* wav_path, int decoder) { @@ -773,7 +810,7 @@ extern "C" char* parakeet_capi_transcribe_path_json_vad_with(parakeet_ctx* ctx, pk::VadRequest req; std::string err; if (!pk::parse_vad_options(options_json, req, err, - vad_ctx ? pk::VadKind::kSilero : pk::VadKind::kHead)) { + vad_ctx ? pk::VadKind::kSilero : pk::VadKind::kHead, /*allow_filter=*/true)) { ctx->last_error = err; return nullptr; } @@ -793,7 +830,7 @@ extern "C" char* parakeet_capi_transcribe_path_json_vad_with(parakeet_ctx* ctx, } pk::SegmenterOpts so = req.opts; pk::Transcription tr = ctx->model->transcribe_pcm_vad_with_timestamps( - audio.samples, audio.sample_rate, to_decoder(decoder), "", so, vad_ctx ? &fn : nullptr); + audio.samples, audio.sample_rate, to_decoder(decoder), "", so, vad_ctx ? &fn : nullptr, req.filter); const pk::ParakeetConfig& cfg = ctx->model->config(); const float frame_sec = (float)cfg.hop_length * (float)cfg.subsampling_factor / (float)cfg.sample_rate; diff --git a/src/transcription_json.cpp b/src/transcription_json.cpp index 7d81de5..4509259 100644 --- a/src/transcription_json.cpp +++ b/src/transcription_json.cpp @@ -88,7 +88,14 @@ std::string transcription_to_json(const Transcription& tr, float frame_sec) { append_json_float(out, "%.4f", tr.tokens[i].conf); out += '}'; } - out += "]}"; + out += ']'; + // Present only when a word filter ran: how many words it removed. + if (tr.dropped_words >= 0) { + out += ",\"guard\":{\"dropped_words\":"; + append_json_int(out, tr.dropped_words); + out += '}'; + } + out += '}'; return out; } diff --git a/src/transcription_json.hpp b/src/transcription_json.hpp index 1bda990..0413572 100644 --- a/src/transcription_json.hpp +++ b/src/transcription_json.hpp @@ -14,7 +14,8 @@ void append_json_int(std::string& out, int v); void append_json_float(std::string& out, const char* fmt, float v); // Serialize a Transcription to the C-API JSON document shape: -// {"text", "frame_sec", "words", "tokens"}. +// {"text", "frame_sec", "words", "tokens"} and, when a word filter ran, +// "guard":{"dropped_words":N}. std::string transcription_to_json(const Transcription& tr, float frame_sec); // Serialize the offline TDT beam result: diff --git a/src/vad_json.cpp b/src/vad_json.cpp index 9d3ce06..ecf5650 100644 --- a/src/vad_json.cpp +++ b/src/vad_json.cpp @@ -47,64 +47,96 @@ bool positive_seconds(double v) { return std::isfinite(v) && v > 0.0 && v <= 1e6 } // namespace -bool parse_vad_options(const char* json, VadRequest& req, std::string& err, VadKind kind) { - req = VadRequest(); - req.kind = kind; - req.opts = default_segmenter_opts(kind); +namespace { + +// Shared parser of the flat option objects. `vad_keys` accepts the segmenter +// and mode keys, `filter_keys` the word filter keys ("trim" belongs to the +// segmenter). `what` names the document in error messages. +bool parse_options(const char* json, VadRequest& req, std::string& err, bool vad_keys, + bool filter_keys, const char* what) { + const std::string docs = std::string(what) + "s"; // "VAD options" / "options" if (!json) return true; Cursor c{json}; c.ws(); if (*c.p == '\0') return true; - if (*c.p != '{') { err = "invalid VAD options: expected a JSON object"; return false; } + if (*c.p != '{') { err = "invalid " + docs + ": expected a JSON object"; return false; } ++c.p; c.ws(); - if (*c.p == '}') { ++c.p; c.ws(); if (*c.p) { err = "invalid VAD options: trailing text"; return false; } return true; } + if (*c.p == '}') { ++c.p; c.ws(); if (*c.p) { err = "invalid " + docs + ": trailing text"; return false; } return true; } for (;;) { c.ws(); std::string key; - if (!parse_string(c, key)) { err = "invalid VAD options: expected a key"; return false; } + if (!parse_string(c, key)) { err = "invalid " + docs + ": expected a key"; return false; } c.ws(); - if (*c.p != ':') { err = "invalid VAD options: expected ':' after \"" + key + "\""; return false; } + if (*c.p != ':') { err = "invalid " + docs + ": expected ':' after \"" + key + "\""; return false; } ++c.p; c.ws(); - if (key == "mode") { + const std::string opt = std::string(what) + " " + key; + const bool is_seg_num = key == "threshold" || key == "min_pause" || key == "min_speech" || + key == "max_segment" || key == "speech_pad" || key == "trim"; + if (vad_keys && key == "mode") { std::string v; - if (!parse_string(c, v)) { err = "invalid VAD option mode: expected a string"; return false; } + if (!parse_string(c, v)) { err = "invalid " + opt + ": expected a string"; return false; } if (v == "speech") req.mode = VadRequest::Mode::kSpeech; else if (v == "segments") req.mode = VadRequest::Mode::kSegments; - else { err = "invalid VAD option mode: use \"speech\" or \"segments\""; return false; } - } else if (key == "probabilities") { - if (std::strncmp(c.p, "true", 4) == 0) { req.probabilities = true; c.p += 4; } - else if (std::strncmp(c.p, "false", 5) == 0) { req.probabilities = false; c.p += 5; } - else { err = "invalid VAD option probabilities: expected true or false"; return false; } - } else if (key == "threshold" || key == "min_pause" || key == "min_speech" || - key == "max_segment" || key == "speech_pad") { + else { err = "invalid " + opt + ": use \"speech\" or \"segments\""; return false; } + } else if ((vad_keys && key == "probabilities") || (filter_keys && key == "drop_punct_only")) { + bool v = false; + if (std::strncmp(c.p, "true", 4) == 0) { v = true; c.p += 4; } + else if (std::strncmp(c.p, "false", 5) == 0) { v = false; c.p += 5; } + else { err = "invalid " + opt + ": expected true or false"; return false; } + if (key == "probabilities") req.probabilities = v; + else req.filter.drop_punct_only = v; + } else if ((vad_keys && is_seg_num) || (filter_keys && (key == "min_local_conf" || key == "local_radius"))) { double v = 0.0; - if (!parse_number(c, v)) { err = "invalid VAD option " + key + ": expected a number"; return false; } + if (!parse_number(c, v)) { err = "invalid " + opt + ": expected a number"; return false; } if (key == "threshold") { - if (!(v > 0.0 && v <= 1.0)) { err = "invalid VAD option threshold: must be in (0, 1]"; return false; } + if (!(v > 0.0 && v <= 1.0)) { err = "invalid " + opt + ": must be in (0, 1]"; return false; } req.opts.threshold = (float)v; - } else if (key == "speech_pad") { - if (!(std::isfinite(v) && v >= 0.0 && v <= 1e6)) { err = "invalid VAD option speech_pad: must be a number of seconds >= 0"; return false; } - req.opts.pad_sec = v; + } else if (key == "min_local_conf") { + if (!(v >= 0.0 && v <= 1.0)) { err = "invalid " + opt + ": must be in [0, 1] (0 = off)"; return false; } + req.filter.min_local_conf = (float)v; + } else if (key == "local_radius") { + if (!positive_seconds(v)) { err = "invalid " + opt + ": must be a number of seconds > 0"; return false; } + req.filter.local_radius_sec = (float)v; + } else if (key == "speech_pad" || key == "trim") { + if (!(std::isfinite(v) && v >= 0.0 && v <= 1e6)) { err = "invalid " + opt + ": must be a number of seconds >= 0"; return false; } + (key == "trim" ? req.opts.trim_sec : req.opts.pad_sec) = v; } else { - if (!positive_seconds(v)) { err = "invalid VAD option " + key + ": must be a number of seconds > 0"; return false; } + if (!positive_seconds(v)) { err = "invalid " + opt + ": must be a number of seconds > 0"; return false; } if (key == "min_pause") req.opts.min_pause_sec = v; else if (key == "min_speech") req.opts.min_speech_sec = v; else req.opts.max_seg_sec = v; } } else { - err = "unknown VAD option: " + key; + err = std::string("unknown ") + what + ": " + key; return false; } c.ws(); if (*c.p == ',') { ++c.p; continue; } if (*c.p == '}') { ++c.p; break; } - err = "invalid VAD options: expected ',' or '}'"; + err = "invalid " + docs + ": expected ',' or '}'"; return false; } c.ws(); - if (*c.p) { err = "invalid VAD options: trailing text"; return false; } + if (*c.p) { err = "invalid " + docs + ": trailing text"; return false; } + return true; +} + +} // namespace + +bool parse_vad_options(const char* json, VadRequest& req, std::string& err, VadKind kind, + bool allow_filter) { + req = VadRequest(); + req.kind = kind; + req.opts = default_segmenter_opts(kind); + return parse_options(json, req, err, true, allow_filter, "VAD option"); +} + +bool parse_filter_options(const char* json, WordFilter& filter, std::string& err) { + VadRequest req; + if (!parse_options(json, req, err, false, true, "option")) return false; + filter = req.filter; return true; } diff --git a/src/vad_json.hpp b/src/vad_json.hpp index db38a0d..5d62423 100644 --- a/src/vad_json.hpp +++ b/src/vad_json.hpp @@ -5,6 +5,7 @@ #include #include +#include "transcription.hpp" #include "vad_segmenter.hpp" namespace pk { @@ -18,16 +19,26 @@ struct VadRequest { Mode mode = Mode::kSpeech; bool probabilities = false; // add "probabilities" to the document VadKind kind = VadKind::kHead; // which model the defaults in `opts` are for + WordFilter filter; // word filter keys, only with allow_filter }; // Parses an options document: a flat JSON object, or NULL / "" for the // defaults. Keys: "threshold" (0 < x <= 1), "min_pause", "min_speech", -// "max_segment" (seconds, > 0), "mode" ("speech" or "segments"), -// "probabilities" (bool), "speech_pad" (seconds >= 0, "speech" mode). Unknown -// keys and bad values are errors. The values that a key leaves out come from -// default_segmenter_opts(kind). Returns false and sets `err` on failure. +// "max_segment" (seconds, > 0), "trim" (seconds >= 0, "segments" mode and the +// transcribe functions; 0 = no trimming), "mode" ("speech" or "segments"), +// "probabilities" (bool), "speech_pad" (seconds >= 0, "speech" mode). With +// `allow_filter` the word filter keys of parse_filter_options are accepted too +// and stored in `req.filter`. Unknown keys and bad values are errors. The +// values that a key leaves out come from default_segmenter_opts(kind). Returns +// false and sets `err` on failure. bool parse_vad_options(const char* json, VadRequest& req, std::string& err, - VadKind kind = VadKind::kHead); + VadKind kind = VadKind::kHead, bool allow_filter = false); + +// The word filter options alone: a flat JSON object, or NULL / "" for a filter +// that is off. Keys: "min_local_conf" (0 <= x <= 1, 0 = off), "local_radius" +// (seconds > 0, default 5), "drop_punct_only" (bool). Unknown keys and bad +// values are errors. Returns false and sets `err` on failure. +bool parse_filter_options(const char* json, WordFilter& filter, std::string& err); // Same document for a Silero VAD model. `pcm` is mono at `sample_rate`, which // must be 16000 or 8000 (the caller resamples anything else); times are on the diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a15a216..c360406 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -250,10 +250,12 @@ pk_add_test(test_vad_options) pk_add_test(test_capi_vad_silero) pk_add_test(test_transcribe_vad_silero) pk_add_test(test_vad_only) +pk_add_test(test_vad_trim_filter) target_compile_definitions(test_silero_vad PRIVATE PK_SOURCE_DIR="${CMAKE_SOURCE_DIR}") set_tests_properties(test_silero_vad test_capi_vad_silero test_transcribe_vad_silero PROPERTIES LABELS "model") set_tests_properties(test_capi_vad_silero test_transcribe_vad_silero PROPERTIES WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) set_tests_properties(test_vad_only PROPERTIES LABELS "model") +set_tests_properties(test_vad_trim_filter PROPERTIES LABELS "model" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) pk_add_test(test_bundle) pk_add_test(test_bundle_models) @@ -262,3 +264,18 @@ if(PARAKEET_WITH_CED AND PARAKEET_WITH_VOICEDETECT) pk_add_test(test_bundle_full) set_tests_properties(test_bundle_full PROPERTIES LABELS "model" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) endif() + +# Option parsing of the trim and word filter flags: each bad value exits at +# once, before any model is read. +if(TARGET parakeet-cli) + function(pk_cli_flag_test name pattern) + add_test(NAME ${name} COMMAND parakeet-cli transcribe --model none.gguf --input none.wav ${ARGN}) + set_tests_properties(${name} PROPERTIES PASS_REGULAR_EXPRESSION "${pattern}") + endfunction() + pk_cli_flag_test(cli_min_local_conf_range "--min-local-conf must be in" --min-local-conf 1.5) + pk_cli_flag_test(cli_min_local_conf_nan "--min-local-conf must be in" --min-local-conf abc) + pk_cli_flag_test(cli_local_radius_zero "--local-radius must be > 0" --local-radius 0) + pk_cli_flag_test(cli_vad_trim_negative "--vad-trim must be >= 0" --vad-trim -1) + pk_cli_flag_test(cli_filter_with_beam "word filter works with greedy decoding only" --min-local-conf 0.5 --beam-size 4) + pk_cli_flag_test(cli_filter_with_stream "word filter is offline only" --drop-punct-only --stream) +endif() diff --git a/tests/test_vad_options.cpp b/tests/test_vad_options.cpp index f4eadb6..b7b894c 100644 --- a/tests/test_vad_options.cpp +++ b/tests/test_vad_options.cpp @@ -64,6 +64,59 @@ int main() { } } + // "trim": default 0.3 for both kinds, 0 turns it off, negative is an error. + CHECK(parse(nullptr, VadKind::kHead, r, err) && near(r.opts.trim_sec, 0.3)); + CHECK(parse(nullptr, VadKind::kSilero, r, err) && near(r.opts.trim_sec, 0.3)); + CHECK(parse("{\"trim\":0}", VadKind::kHead, r, err) && r.opts.trim_sec == 0.0 && near(r.opts.max_seg_sec, 30.0)); + CHECK(parse("{\"trim\":0.5,\"mode\":\"segments\"}", VadKind::kSilero, r, err) && near(r.opts.trim_sec, 0.5) && + r.mode == VadRequest::Mode::kSegments && near(r.opts.pad_sec, 0.03)); + for (VadKind k : {VadKind::kHead, VadKind::kSilero}) { + CHECK(!parse("{\"trim\":-0.1}", k, r, err) && err.find("trim") != std::string::npos); + CHECK(!parse("{\"trim\":\"x\"}", k, r, err) && err.find("trim") != std::string::npos); + CHECK(!parse("{\"trim\":1e9}", k, r, err) && err.find("trim") != std::string::npos); + } + + // Word filter keys: only with allow_filter; off by default. + CHECK(parse(nullptr, VadKind::kHead, r, err) && !r.filter.active() && near(r.filter.local_radius_sec, 5.0)); + CHECK(!parse_vad_options("{\"min_local_conf\":0.5}", r, err, VadKind::kHead, false) && + err.find("min_local_conf") != std::string::npos); + CHECK(!parse_vad_options("{\"drop_punct_only\":true}", r, err, VadKind::kHead, false) && + err.find("drop_punct_only") != std::string::npos); + CHECK(parse_vad_options("{\"min_local_conf\":0.5,\"local_radius\":3,\"drop_punct_only\":true,\"trim\":0.2,\"max_segment\":10}", + r, err, VadKind::kSilero, true)); + CHECK(r.filter.min_local_conf == 0.5f && r.filter.local_radius_sec == 3.0f && r.filter.drop_punct_only && + r.filter.active() && near(r.opts.trim_sec, 0.2) && near(r.opts.max_seg_sec, 10.0)); + CHECK(parse_vad_options("{\"min_local_conf\":0,\"drop_punct_only\":false}", r, err, VadKind::kHead, true) && !r.filter.active()); + CHECK(parse_vad_options("{\"min_local_conf\":1}", r, err, VadKind::kHead, true) && r.filter.min_local_conf == 1.0f); + { + const struct { const char* json; const char* word; } badf[] = { + {"{\"min_local_conf\":1.01}", "min_local_conf"}, {"{\"min_local_conf\":-0.5}", "min_local_conf"}, + {"{\"min_local_conf\":\"x\"}", "min_local_conf"}, {"{\"local_radius\":0}", "local_radius"}, + {"{\"local_radius\":-1}", "local_radius"}, {"{\"local_radius\":1e9}", "local_radius"}, + {"{\"drop_punct_only\":1}", "drop_punct_only"}, {"{\"drop_punct_only\":\"true\"}", "drop_punct_only"}, + }; + for (const auto& b : badf) { + CHECK(!parse_vad_options(b.json, r, err, VadKind::kHead, true)); + CHECK(err.find(b.word) != std::string::npos); + } + } + // parse_filter_options: the filter keys alone. + { + WordFilter f; + CHECK(parse_filter_options(nullptr, f, err) && !f.active()); + CHECK(parse_filter_options("", f, err) && !f.active()); + CHECK(parse_filter_options("{}", f, err) && !f.active() && f.local_radius_sec == 5.0f); + CHECK(parse_filter_options("{\"min_local_conf\":0.5}", f, err) && f.min_local_conf == 0.5f && f.local_radius_sec == 5.0f && + !f.drop_punct_only); + CHECK(parse_filter_options("{\"drop_punct_only\":true,\"local_radius\":2.5}", f, err) && f.drop_punct_only && + f.min_local_conf == 0.0f && f.local_radius_sec == 2.5f); + const char* bad_only[] = {"{\"threshold\":0.5}", "{\"trim\":0.3}", "{\"mode\":\"speech\"}", "{\"max_segment\":5}", + "{\"nope\":1}", "[1]", "{\"min_local_conf\":", "{} x", "{\"min_local_conf\":2}"}; + for (const char* j : bad_only) CHECK(!parse_filter_options(j, f, err) && !err.empty()); + CHECK(!parse_filter_options("{\"threshold\":0.5}", f, err) && err.find("threshold") != std::string::npos && + err.find("unknown") != std::string::npos); + } + // C-API: NULL and wrong-kind contexts fail without a crash. CHECK(parakeet_capi_vad_pcm_json(nullptr, nullptr, 0, 16000, nullptr) == nullptr); CHECK(parakeet_capi_vad_path_json(nullptr, "x.wav", nullptr) == nullptr); @@ -72,6 +125,7 @@ int main() { CHECK(parakeet_capi_vad_stream_reset(nullptr) != 0); parakeet_capi_vad_stream_free(nullptr); CHECK(parakeet_capi_transcribe_path_json_vad_with(nullptr, nullptr, "x.wav", 0, nullptr) == nullptr); + CHECK(parakeet_capi_transcribe_path_json_with(nullptr, "x.wav", 0, nullptr) == nullptr); CHECK(parakeet_capi_model_kind(nullptr) == PARAKEET_MODEL_KIND_NONE); CHECK(parakeet_capi_load("does-not-exist.gguf") == nullptr); diff --git a/tests/test_vad_trim_filter.cpp b/tests/test_vad_trim_filter.cpp new file mode 100644 index 0000000..aabf88b --- /dev/null +++ b/tests/test_vad_trim_filter.cpp @@ -0,0 +1,331 @@ +// VAD segment trimming and the word filter on real models. +// +// Clip: fixtures joined by long stretches of digital silence, so the VAD finds +// pauses. Checks, per model: +// * trimmed segments carry less audio, and the words and their times match the +// untrimmed decode of the same speech (offsets are right); +// * trim 0 gives the same Transcription as before trimming existed (the +// segments of the legacy segmenter, decoded one by one); +// * the word filter off is byte-identical: Model call, C-API JSON; +// * the filter on: guard member, words removed from text, words and tokens. +// Env (each optional; skip 77 when none is set): +// PARAKEET_TEST_GGUF_ULTRA / _REDUX_KEEP / _REDUX_DEQ models with a VAD head +// PARAKEET_TEST_GGUF any ASR model (with PARAKEET_TEST_SILERO_GGUF: Silero cuts) +// PARAKEET_TEST_GGUF_CTC a CTC model: the filter keys on the CTC head +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "audio_io.hpp" +#include "model.hpp" +#include "parakeet_capi.h" +#include "silero_vad.hpp" + +using namespace pk; + +static int failures = 0; +#define CHECK(c) do { if (!(c)) { std::fprintf(stderr, "FAIL: %s (line %d)\n", #c, __LINE__); ++failures; } } while (0) + +static std::vector words_of(const std::string& s) { + std::istringstream is(s); + std::vector w; + for (std::string x; is >> x;) { + std::string y; + for (char c : x) if (std::isalnum((unsigned char)c)) y += (char)std::tolower((unsigned char)c); + if (!y.empty()) w.push_back(y); + } + return w; +} + +static bool same_f(float a, float b) { return std::memcmp(&a, &b, sizeof(float)) == 0; } +static bool same_tr(const Transcription& a, const Transcription& b) { + if (a.text != b.text || a.words.size() != b.words.size() || a.tokens.size() != b.tokens.size() || + a.dropped_words != b.dropped_words) + return false; + for (size_t i = 0; i < a.words.size(); ++i) + if (a.words[i].text != b.words[i].text || !same_f(a.words[i].start, b.words[i].start) || + !same_f(a.words[i].end, b.words[i].end) || !same_f(a.words[i].conf, b.words[i].conf)) + return false; + for (size_t i = 0; i < a.tokens.size(); ++i) + if (a.tokens[i].id != b.tokens[i].id || a.tokens[i].frame != b.tokens[i].frame || + a.tokens[i].span != b.tokens[i].span || !same_f(a.tokens[i].conf, b.tokens[i].conf)) + return false; + return true; +} + +// speech.wav, 24 s of digital silence, speech.wav, 24 s of silence: about 62 s. +static Audio make_clip(double* first_end, double* second_start) { + Audio sp; + CHECK(load_audio_16k_mono("tests/fixtures/speech.wav", sp)); + Audio clip; + clip.sample_rate = 16000; + const std::vector gap((size_t)(24 * 16000), 0.0f); + clip.samples.insert(clip.samples.end(), sp.samples.begin(), sp.samples.end()); + *first_end = (double)clip.samples.size() / 16000.0; + clip.samples.insert(clip.samples.end(), gap.begin(), gap.end()); + *second_start = (double)clip.samples.size() / 16000.0; + clip.samples.insert(clip.samples.end(), sp.samples.begin(), sp.samples.end()); + clip.samples.insert(clip.samples.end(), gap.begin(), gap.end()); + return clip; +} + +static size_t edit_distance(const std::vector& a, const std::vector& b) { + std::vector prev(b.size() + 1), cur(b.size() + 1); + for (size_t j = 0; j <= b.size(); ++j) prev[j] = j; + for (size_t i = 1; i <= a.size(); ++i) { + cur[0] = i; + for (size_t j = 1; j <= b.size(); ++j) + cur[j] = std::min({prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (a[i - 1] == b[j - 1] ? 0 : 1)}); + std::swap(prev, cur); + } + return prev[b.size()]; +} + +static double covered(const std::vector& s) { + double t = 0.0; + for (const auto& g : s) t += g.end - g.start; + return t; +} + +// `fn` set: Silero probabilities (opts are Silero's); else the model's own head. +static void check_trim(const Model& m, const Audio& clip, double first_end, double second_start, + SegmenterOpts opts, const Model::VadProbabilityFn* fn, const char* what, bool strict, + bool quiet_gap) { + std::fprintf(stderr, " %s: trim\n", what); + const double total = (double)clip.samples.size() / 16000.0; + SegmenterOpts so = opts; + if (!fn) so.frame_sec = m.config().vad.frame_sec; + const std::vector p = fn ? (*fn)(clip.samples) : m.vad_probabilities(clip.samples); + SegmenterOpts old = so; + old.trim_sec = 0.0; + const auto segs_new = segment_by_vad(p, total, so); + const auto segs_old = segment_by_vad(p, total, old); + std::fprintf(stderr, " %zu segments, %.1f s with trim, %.1f s without\n", segs_new.size(), + covered(segs_new), covered(segs_old)); + CHECK(segs_new.size() >= 2 && segs_new.size() == segs_old.size()); + CHECK(covered(segs_new) < covered(segs_old)); + // The detector calls the digital silence between the speech non-speech: only + // the speech is decoded. (The Redux head fires on digital silence, so it does + // not get this check.) + if (quiet_gap) { + CHECK(covered(segs_new) < covered(segs_old) - 10.0); + CHECK(covered(segs_new) < 2.0 * first_end + 4 * so.trim_sec + 1.0); + } + + // Word times: the second copy of the speech is the first one shifted. + opts.trim_sec = so.trim_sec; + const Transcription tr = m.transcribe_pcm_vad_with_timestamps(clip.samples, 16000, Decoder::kDefault, "", opts, fn); + opts.trim_sec = 0.0; + const Transcription tr0 = m.transcribe_pcm_vad_with_timestamps(clip.samples, 16000, Decoder::kDefault, "", opts, fn); + CHECK(tr.dropped_words == -1 && tr0.dropped_words == -1); + CHECK(!tr.words.empty()); + // Another cut changes the audio the decoder sees, so a model may differ by a + // word or two; the head model with these fixtures does not. + if (strict) { + CHECK(words_of(tr.text) == words_of(tr0.text)); + CHECK(tr.words.size() == tr0.words.size()); + } else { + const auto a = words_of(tr.text), b = words_of(tr0.text); + CHECK(edit_distance(a, b) <= 1 + b.size() / 10); + } + float prev = -1.0f; + for (const Word& w : tr.words) { + CHECK(w.start >= prev - 1e-3f && w.end >= w.start && w.end <= (float)total + 0.01f); + prev = w.start; + } + for (size_t i = 1; i < tr.tokens.size(); ++i) CHECK(tr.tokens[i].frame >= tr.tokens[i - 1].frame); + for (size_t i = 0; strict && i < tr.words.size() && i < tr0.words.size(); ++i) { + // Trimming moves the start of a segment, and with it the 80 ms frame grid + // of the encoder: times agree to a few frames. + CHECK(std::fabs(tr.words[i].start - tr0.words[i].start) < 0.5f); + } + // The first words are in the first speech, the last words in the second. + if (!tr.words.empty()) { + CHECK(tr.words.front().start < (float)first_end); + CHECK(tr.words.back().start > (float)second_start); + size_t in_gap = 0; + for (const Word& w : tr.words) in_gap += (w.start > (float)first_end + 0.5f && w.start < (float)second_start - 0.5f); + if (quiet_gap) CHECK(in_gap == 0); + } + // The text path matches the timestamp path. + CHECK(m.transcribe_pcm_vad(clip.samples, 16000, Decoder::kDefault, "", opts, fn) == tr0.text); +} + +static void check_filter(const Model& m, const char* wav_path, const Audio& clip, const SegmenterOpts& opts, + const Model::VadProbabilityFn* fn, const char* what) { + std::fprintf(stderr, " %s: filter\n", what); + // Off: the filter object changes nothing, on a short clip and on a long one. + Audio sp; + CHECK(load_audio_16k_mono("tests/fixtures/speech.wav", sp)); + { + const Transcription plain = m.transcribe_with_timestamps(sp.samples, 16000, Decoder::kDefault); + const Transcription off = m.transcribe_pcm_vad_with_timestamps(sp.samples, 16000, Decoder::kDefault, "", opts, fn, WordFilter()); + CHECK(same_tr(plain, off) && off.dropped_words == -1); + } + const Transcription base = m.transcribe_pcm_vad_with_timestamps(clip.samples, 16000, Decoder::kDefault, "", opts, fn); + CHECK(same_tr(base, m.transcribe_pcm_vad_with_timestamps(clip.samples, 16000, Decoder::kDefault, "", opts, fn, WordFilter()))); + CHECK(!base.words.empty()); + + // A mild filter keeps clean speech as it is; the guard member says nothing was dropped. + WordFilter mild; + mild.min_local_conf = 0.5f; + const Transcription kept = m.transcribe_pcm_vad_with_timestamps(clip.samples, 16000, Decoder::kDefault, "", opts, fn, mild); + CHECK(kept.dropped_words == 0); + Transcription expect = base; + expect.dropped_words = 0; + CHECK(same_tr(kept, expect)); + // The text path with a filter gives the filtered text. + CHECK(m.transcribe_pcm_vad(clip.samples, 16000, Decoder::kDefault, "", opts, fn, mild) == kept.text); + + // A threshold of 1 drops every word (no word has mean confidence 1), per + // decode unit, and the count adds up. + WordFilter all; + all.min_local_conf = 1.0f; + const Transcription none = m.transcribe_pcm_vad_with_timestamps(clip.samples, 16000, Decoder::kDefault, "", opts, fn, all); + CHECK(none.words.empty() && none.tokens.empty() && none.text.empty()); + CHECK(none.dropped_words == (int)base.words.size()); + // A radius too small to reach a neighbour leaves the own confidence: the + // low words go, the rest stays, and the numbers agree between words and tokens. + WordFilter own; + own.min_local_conf = 0.9f; + own.local_radius_sec = 0.001f; + const Transcription some = m.transcribe_pcm_vad_with_timestamps(clip.samples, 16000, Decoder::kDefault, "", opts, fn, own); + CHECK(some.dropped_words >= 0 && (size_t)some.dropped_words + some.words.size() == base.words.size()); + CHECK(some.tokens.size() <= base.tokens.size()); + for (const Word& w : some.words) CHECK(w.conf >= 0.9f - 1e-6f); + CHECK(words_of(some.text).size() <= base.words.size()); + (void)wav_path; +} + +static char* take_dup(char* p, std::string& out) { + if (p) { out = p; parakeet_capi_free_string(p); } + return p; +} + +static void check_capi(const char* asr_path, const char* silero_path, const char* what) { + std::fprintf(stderr, " %s: C-API\n", what); + parakeet_ctx* ctx = parakeet_capi_load(asr_path); + CHECK(ctx != nullptr); + if (!ctx) return; + const char* wav = "tests/fixtures/speech.wav"; + std::string plain, off, none_opts, kept, dropped; + CHECK(take_dup(parakeet_capi_transcribe_path_json(ctx, wav, 0), plain)); + // No options, empty options, and an explicit off give the plain document, byte for byte. + CHECK(take_dup(parakeet_capi_transcribe_path_json_with(ctx, wav, 0, nullptr), off) && off == plain); + CHECK(take_dup(parakeet_capi_transcribe_path_json_with(ctx, wav, 0, ""), none_opts) && none_opts == plain); + CHECK(take_dup(parakeet_capi_transcribe_path_json_with(ctx, wav, 0, "{}"), none_opts) && none_opts == plain); + CHECK(take_dup(parakeet_capi_transcribe_path_json_with(ctx, wav, 0, "{\"min_local_conf\":0,\"local_radius\":3}"), none_opts) && + none_opts == plain); + CHECK(plain.find("\"guard\"") == std::string::npos); + // On: guard present; 0.5 keeps clean speech; 1 drops every word. + CHECK(take_dup(parakeet_capi_transcribe_path_json_with(ctx, wav, 0, "{\"min_local_conf\":0.5}"), kept)); + CHECK(kept.size() > plain.size() && kept.find("\"guard\":{\"dropped_words\":0}") != std::string::npos); + CHECK(kept.substr(0, plain.size() - 1) == plain.substr(0, plain.size() - 1)); + CHECK(take_dup(parakeet_capi_transcribe_path_json_with(ctx, wav, 0, "{\"min_local_conf\":1}"), dropped)); + CHECK(dropped.find("\"text\":\"\"") != std::string::npos && dropped.find("\"words\":[]") != std::string::npos && + dropped.find("\"tokens\":[]") != std::string::npos && dropped.find("\"guard\":{\"dropped_words\":") != std::string::npos && + dropped.find("\"dropped_words\":0}") == std::string::npos); + // Errors: NULL, bad keys, bad values, the last error says which. + CHECK(parakeet_capi_transcribe_path_json_with(nullptr, wav, 0, nullptr) == nullptr); + CHECK(parakeet_capi_transcribe_path_json_with(ctx, nullptr, 0, nullptr) == nullptr); + CHECK(parakeet_capi_transcribe_path_json_with(ctx, "/nonexistent.wav", 0, nullptr) == nullptr); + const struct { const char* json; const char* word; } bad[] = { + {"{\"min_local_conf\":1.5}", "min_local_conf"}, {"{\"min_local_conf\":-0.1}", "min_local_conf"}, + {"{\"local_radius\":0}", "local_radius"}, {"{\"drop_punct_only\":1}", "drop_punct_only"}, + {"{\"threshold\":0.5}", "threshold"}, // a VAD key is not a filter key + {"{\"nope\":1}", "nope"}, {"[1]", "JSON object"}}; + for (const auto& b : bad) { + CHECK(parakeet_capi_transcribe_path_json_with(ctx, wav, 0, b.json) == nullptr); + CHECK(std::string(parakeet_capi_last_error(ctx)).find(b.word) != std::string::npos); + } + // The VAD variant takes the same filter keys and the trim key. + if (silero_path) { + parakeet_ctx* v = parakeet_capi_load(silero_path); + CHECK(v != nullptr); + if (v) { + std::string a, b, c; + CHECK(take_dup(parakeet_capi_transcribe_path_json_vad_with(ctx, v, wav, 0, nullptr), a)); + CHECK(take_dup(parakeet_capi_transcribe_path_json_vad_with(ctx, v, wav, 0, "{\"trim\":0,\"max_segment\":10}"), b)); + CHECK(take_dup(parakeet_capi_transcribe_path_json_vad_with(ctx, v, wav, 0, "{\"trim\":0.1,\"max_segment\":10,\"min_local_conf\":0.5,\"drop_punct_only\":true}"), c)); + CHECK(a.find("\"guard\"") == std::string::npos && b.find("\"guard\"") == std::string::npos && + c.find("\"guard\":{\"dropped_words\":") != std::string::npos); + CHECK(parakeet_capi_transcribe_path_json_vad_with(ctx, v, wav, 0, "{\"trim\":-1}") == nullptr); + CHECK(std::string(parakeet_capi_last_error(ctx)).find("trim") != std::string::npos); + CHECK(parakeet_capi_transcribe_path_json_vad_with(ctx, v, wav, 0, "{\"min_local_conf\":2}") == nullptr); + parakeet_capi_free(v); + } + } + parakeet_capi_free(ctx); +} + +int main() { + // The Redux head fires on digital silence, so only Ultra gets the "silence is not decoded" checks. + std::vector> heads; + for (const char* e : {"PARAKEET_TEST_GGUF_ULTRA", "PARAKEET_TEST_GGUF_REDUX_KEEP", "PARAKEET_TEST_GGUF_REDUX_DEQ"}) + if (const char* v = std::getenv(e)) heads.push_back({v, std::strstr(e, "ULTRA") != nullptr}); + const char* any = std::getenv("PARAKEET_TEST_GGUF"); + const char* silero = std::getenv("PARAKEET_TEST_SILERO_GGUF"); + const char* ctc = std::getenv("PARAKEET_TEST_GGUF_CTC"); + if (heads.empty() && !(any && silero) && !ctc) { std::puts("skip: no model env set"); return 77; } + + double first_end = 0.0, second_start = 0.0; + const Audio clip = make_clip(&first_end, &second_start); + + for (const auto& h : heads) { + const char* path = h.first; + std::fprintf(stderr, "head model %s\n", path); + std::unique_ptr m = Model::load(path); + if (!m) { CHECK(false); continue; } + if (!m->config().vad.present) continue; + check_trim(*m, clip, first_end, second_start, SegmenterOpts(), nullptr, "head", true, h.second); + check_filter(*m, "", clip, SegmenterOpts(), nullptr, "head"); + check_capi(path, silero, "head"); + } + if (any && silero) { + std::fprintf(stderr, "silero cuts, model %s\n", any); + std::string err; + std::unique_ptr sv = SileroVad::load(silero, &err); + std::unique_ptr m = Model::load(any); + CHECK(sv && m); + if (sv && m) { + const SileroVad* svp = sv.get(); + const Model::VadProbabilityFn fn = [svp](const std::vector& pcm) { + return svp->probabilities(pcm.data(), pcm.size(), 16000); + }; + const SegmenterOpts so = default_segmenter_opts(VadKind::kSilero); + check_trim(*m, clip, first_end, second_start, so, &fn, "silero", false, true); + check_filter(*m, "", clip, so, &fn, "silero"); + check_capi(any, silero, "silero"); + } + } + if (ctc) { + std::fprintf(stderr, "CTC model %s\n", ctc); + std::unique_ptr m = Model::load(ctc); + CHECK(m != nullptr); + if (m) { + Audio sp; + CHECK(load_audio_16k_mono("tests/fixtures/speech.wav", sp)); + Transcription t = m->transcribe_with_timestamps(sp.samples, 16000, Decoder::kCTC); + const Transcription ref = t; + WordFilter f; + f.min_local_conf = 0.5f; + f.drop_punct_only = true; + apply_word_filter(t, f); + CHECK(t.dropped_words == 0 && t.text == ref.text && t.words.size() == ref.words.size()); + f.min_local_conf = 1.0f; + apply_word_filter(t, f); + CHECK(t.words.empty() && t.tokens.empty() && t.dropped_words == (int)ref.words.size()); + } + } + if (failures) return 1; + std::puts("test_vad_trim_filter: OK"); + return 0; +} From c7e15067b05030287b00c2864e4c9db8221b56ee Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 4 Oct 2026 23:56:19 +0000 Subject: [PATCH 4/4] docs: segment trim, word filter, and the runs behind them Describe trim and the opt-in word filter in docs/vad.md and the README. Add a section to docs/vad-benchmarks.md with the word error rates, the seconds of noise decoded and the invented words before and after the trim, and the effect of the filter at 0.5, 0.7 and 0.9. The scripts that fetch the public data, build the noise files with fixed seeds, run the CLI and print the tables are in scripts/vad_bench/decoder_guards, with the printed tables. No audio or model is committed. The trim default changes the transcripts of long audio through the VAD paths slightly, so the docs say so. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- README.md | 2 + docs/vad-benchmarks.md | 84 +++++++++++ docs/vad.md | 63 +++++++- scripts/vad_bench/decoder_guards/README.md | 38 +++++ .../vad_bench/decoder_guards/fetch_data.py | 42 ++++++ .../vad_bench/decoder_guards/make_corpus.py | 99 +++++++++++++ .../decoder_guards/results/tables.txt | 70 +++++++++ scripts/vad_bench/decoder_guards/run_all.py | 93 ++++++++++++ scripts/vad_bench/decoder_guards/tables.py | 138 ++++++++++++++++++ 9 files changed, 627 insertions(+), 2 deletions(-) create mode 100644 scripts/vad_bench/decoder_guards/README.md create mode 100644 scripts/vad_bench/decoder_guards/fetch_data.py create mode 100644 scripts/vad_bench/decoder_guards/make_corpus.py create mode 100644 scripts/vad_bench/decoder_guards/results/tables.txt create mode 100644 scripts/vad_bench/decoder_guards/run_all.py create mode 100644 scripts/vad_bench/decoder_guards/tables.py diff --git a/README.md b/README.md index ceebfa7..3b7d720 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,8 @@ parakeet-cli transcribe ... --beam-size 4 --nbest 4 # TDT N-best, se parakeet-cli transcribe ... --stream # cache-aware streaming (EOU and Nemotron models) parakeet-cli transcribe ... --lang # Nemotron 3.5 language, default auto parakeet-cli transcribe ... --vad [--vad-model silero.gguf] # cut long audio at pauses (offline, greedy only) +parakeet-cli transcribe ... --vad-trim SEC # trim each piece to its speech plus SEC (default 0.3, 0 = whole cuts) +parakeet-cli transcribe ... --min-local-conf 0.5 # opt-in: drop words invented on noise (docs/vad.md) parakeet-cli vad --model M --input A.wav [--mode segments] [--probabilities] # speech regions as JSON parakeet-cli scene --model ASR --diar DIAR --sound CED --input A.wav # words + speakers + sounds parakeet-cli scene ... --speakers SPK.gguf --registry R # name the speakers diff --git a/docs/vad-benchmarks.md b/docs/vad-benchmarks.md index fd91aa5..059e52f 100644 --- a/docs/vad-benchmarks.md +++ b/docs/vad-benchmarks.md @@ -925,6 +925,89 @@ compared with the model runs, but it was not timed separately. Scripts, the small result files and how to reproduce: [`scripts/vad_bench/fusion/`](../scripts/vad_bench/fusion/README.md). +## Trimming segments and the word filter + +Two changes to what `transcribe --vad` hands to the decoder. Scripts and the full output are in +[scripts/vad_bench/decoder_guards](../scripts/vad_bench/decoder_guards) (`results/tables.txt`). The +options are described in [vad.md](vad.md). + +The run was not made on a quiet machine: the load average was between 5 and 50 (see the first line +of `results/tables.txt`). Word error rates and word counts do not depend on load; no timing is +claimed. + +### Setup + +- Old: `parakeet-cli` before the change (the cuts of the previous segmenter). New: the default, + each segment trimmed to its speech plus 0.3 s. `--vad-trim 0` gave the old output byte for byte on + all 45 files that were run both ways (15 per detector). +- Detectors: the Ultra head, the Redux head (dequantized F16), and TDT 0.6B v3 with Silero. + The model was Ultra F16 for the head runs. +- Talks: three whole TED-LIUM long-form talks that were not used to choose any setting (5627 + reference words). Words are lower-cased, punctuation removed. +- Speech in noise: 4 sets of 6 LibriSpeech test-clean utterances with gaps (about 50 s each, so the + segmenter cuts them), clean, with white noise at 5 dB SNR, or with pink noise at 0 dB SNR. 429 + reference words per condition. +- Noise block: 90 s of a talk with 60 s of synthetic noise (white, pink, clicks, music-like tones) put + in at a quiet point, 30 files. Any word the decoder returns inside the block is an invented word. + "Seconds decoded" is the overlap of the cuts with the block, from `vad --mode segments`. +- Noise alone: 63 files of 30 s (seven noise types, three levels), decoded whole. + +### Trimming: word error rate + +| Set | Detector | Old | Trim 0.3 | Change | +| --- | --- | ---: | ---: | ---: | +| 3 talks | Ultra head | 3.68 | 3.68 | +0.00 | +| 3 talks | Redux head | 4.39 | 4.51 | +0.12 | +| 3 talks | v3 + Silero | 3.45 | 3.45 | +0.00 | +| clean | Ultra / Redux / v3 + Silero | 2.10 / 3.26 / 1.86 | 1.86 / 3.03 / 1.86 | -0.23 / -0.23 / +0.00 | +| white noise 5 dB | Ultra / Redux / v3 + Silero | 4.90 / 7.23 / 5.59 | 4.43 / 6.76 / 5.13 | -0.47 / -0.47 / -0.47 | +| pink noise 0 dB | Ultra / Redux / v3 + Silero | 6.53 / 12.35 / 7.93 | 5.59 / 13.29 / 7.69 | -0.93 / +0.93 / -0.23 | + +On talks the change is within 0.12 points; the Redux head loses a little on two of the three talks +and on pink noise. The speech-in-noise sets are small (one word is 0.23 points), so read them as +"neutral", not as a gain. + +### Trimming: the noise block + +| Detector | Seconds of the 60 s block decoded, old | New | Invented words, old | New | +| --- | ---: | ---: | ---: | ---: | +| Ultra head | 33.8 | 12.0 | 0 | 0 | +| Redux head | 29.7 | 0.3 | 0 | 0 | +| v3 + Silero | 27.7 | 0.1 | 11 (5 files) | 0 | + +The seconds are means over the 30 files. The head still passes loud white and pink noise (-5 dB +against the speech) as speech, and then the trim cannot remove it: Ultra decodes 52 to 58 s of +those blocks. The Redux head also fires on digital silence. The words the decoder invented were +rare in this run (Ultra and Redux gave none, in the block or in 63 noise-only files), so the +trim's benefit here is mostly seconds saved, and for v3 the 11 invented words. + +### Word filter at 0.5 + +| Check | Ultra head | v3 + Silero | +| --- | ---: | ---: | +| WER on the 3 talks, trim 0.3 / plus filter | 3.68 / 3.68 | 3.45 / 3.45 | +| WER on speech in noise (12 files, 1287 words): off / 0.5 / 0.7 / 0.9 | 3.96 / 3.96 / 3.96 / 6.06 | 4.90 / 4.90 / 4.90 / 16.08 | +| Words dropped at 0.9 on those files | 31 | 164 | +| Words dropped at 0.5 on all files with real speech | 0 of 15375 | 3 of 15406 | +| Invented words in the 5 blocks that had them (old cuts, filter alone) | no events | 11 -> 1, 0 of 1484 other words lost | +| Invented words in the 63 noise-only files | 0 | 1 -> 0 (v3) | + +At 0.5 the filter is free on these sets and it removes most of the few invented words there were. +At 0.9 it costs real words, most on v3. Hence 0.5 is the suggested value. + +### Limits + +- The noise is synthetic. The event counts are small: 11 invented words in 5 files, all from v3. A + filter that removes 10 of 11 is a weak estimate of a rate. +- The filter was not run against invented words on Ultra, Redux or an RNN-T model (none occurred), and + not at all on a CTC model of 0.6B (a unit test runs it on one fixture). The `drop_punct_only` + option for CTC rests on the 110M CTC head, not measured here. +- A single lone invented word between real speech was not tested. +- Silero followed by the head (the two-stage rule), GPU backends and the quantized files other than the + F16 ones were not tested. +- The trim changes transcripts of long audio through the VAD paths slightly; the numbers above are + from three talks and 12 clips per condition. + ## When to use which This is limited to what the numbers above support. @@ -977,6 +1060,7 @@ are committed. In short: `silero_collect.py`, `silero_analyze.py`; speed with `speed_silero.py`. 5. Long talks: `longform_b1.sh`. 6. Noise root-cause study: [`noise_dive/`](../scripts/vad_bench/noise_dive/README.md). +7. Segment trim and word filter: [`decoder_guards/`](../scripts/vad_bench/decoder_guards/README.md). Small result files of the runs on this page (tables, per run timings and load logs) are in `scripts/vad_bench/results/`. The raw per clip predictions are not committed; diff --git a/docs/vad.md b/docs/vad.md index 5388a08..c7da89d 100644 --- a/docs/vad.md +++ b/docs/vad.md @@ -98,6 +98,7 @@ the same keys). Unknown keys and out of range values are errors. | `min_speech` | seconds; shorter speech runs are dropped | 0.1 | 0.25 | | `speech_pad` | seconds >= 0, `speech` mode: widen each region on both sides | 0 | 0.03 | | `max_segment` | seconds; cap in `segments` mode | 30 | 30 | +| `trim` | seconds >= 0; `segments` mode and the transcribe functions: shrink each cut to its speech plus this much on each side, 0 = keep the whole cut | 0.3 | 0.3 | | `mode` | `speech` or `segments` | `speech` | `speech` | | `probabilities` | add the per frame probabilities | false | false | @@ -106,7 +107,64 @@ shorter than 0.1 s are bridged, runs shorter than `min_speech` are dropped, regions closer than `min_pause` merge, then each region is padded (and two regions that would overlap meet in the middle of the gap). `segments` is the cut that `transcribe --vad` decodes: pieces of at most `max_segment` seconds cut at -pauses, pieces without speech dropped, audio within the cap returned whole. +pauses, pieces without speech dropped, audio within the cap returned whole. Each +kept piece is then trimmed (see below). + +### Trimming the segments + +A piece cut from long audio used to carry everything between its two cut points +to the decoder, including the noise and silence the VAD had already flagged as +non-speech. That is where an ASR model invents words. Since this change each +piece shrinks to its first speech frame minus `trim` and its last speech frame +plus `trim` (default 0.3 s, never past the cut itself). Speech is the smoothed +mask, so the rules above still decide what counts as speech. Audio within the +cap is not cut and not trimmed. Word and token times are still relative to the +whole file. `trim` 0 (`--vad-trim 0`) gives the previous cuts exactly. + +This is a change of default behaviour for `transcribe --vad`, +`parakeet_capi_transcribe_path_json_vad*` and the `segments` mode of the VAD +functions, for the head, for Silero and for VAD-only slices (they share the +segmenter). On talks, transcripts of long audio can shift slightly (a word WER +cost of about 0.1 point in our runs); on audio with long noisy stretches the +decoder sees much less noise. Numbers: [vad-benchmarks.md](vad-benchmarks.md#trimming-segments-and-the-word-filter). + +## Word filter (opt-in) + +A confidence filter can remove the words a model invents on noise. It is +post-processing of the decode: the model, the VAD and the cuts are unchanged, and +it is off by default (output is then byte for byte the same). + +A word is dropped when the mean confidence of the words that start within +`local_radius` seconds of it (the word itself included, the same decode unit) is +below `min_local_conf`. A low confidence word between confident ones keeps a high +mean and stays; a word that stands alone, or among other low confidence words, +goes. A decode unit is the whole clip, or one VAD segment. `drop_punct_only` also +removes words that consist only of punctuation. + +| Option | Meaning | Default | +| --- | --- | --- | +| `min_local_conf` | 0 to 1; 0 = off. 0.5 is the suggested value | 0 | +| `local_radius` | seconds, both sides | 5 | +| `drop_punct_only` | drop words with no letter or digit; recommended for CTC models | false | + +``` +parakeet-cli transcribe --model m.gguf --input a.wav --min-local-conf 0.5 \ + [--local-radius SEC] [--drop-punct-only] [--json] [--vad ...] + +char* parakeet_capi_transcribe_path_json_with(parakeet_ctx* ctx, const char* wav_path, + int decoder, const char* options_json); +``` + +The options JSON of `parakeet_capi_transcribe_path_json_with` takes the three +keys above. `parakeet_capi_transcribe_path_json_vad_with` takes them too, next to +the VAD keys (`trim` included). With a filter on, the JSON document gets one more +member, `"guard":{"dropped_words":N}` (N is 0 when nothing was dropped), and the +dropped words are also removed from `text`, `words` and `tokens`. + +Limits: 0.5 removes hallucinated words on Ultra, Redux and RNN-T models at no +cost in WER on clean speech. On v3 and on CTC models, and at higher thresholds, +it also removes real words (see the benchmark page). It does not save time: the +decoder still runs on everything it is given. ### Why the Silero defaults differ @@ -167,7 +225,8 @@ audio with Silero: ``` parakeet-cli transcribe --model tdt-0.6b-v3.gguf --input long.wav --vad --vad-model silero.gguf \ - [--vad-threshold F] [--vad-min-pause SEC] [--vad-min-speech SEC] [--vad-max-seg SEC] + [--vad-threshold F] [--vad-min-pause SEC] [--vad-min-speech SEC] [--vad-max-seg SEC] \ + [--vad-trim SEC] char* parakeet_capi_transcribe_path_json_vad_with(parakeet_ctx* asr, parakeet_ctx* silero, const char* wav_path, int decoder, diff --git a/scripts/vad_bench/decoder_guards/README.md b/scripts/vad_bench/decoder_guards/README.md new file mode 100644 index 0000000..9fa8198 --- /dev/null +++ b/scripts/vad_bench/decoder_guards/README.md @@ -0,0 +1,38 @@ +# Trim and word filter runs + +Scripts behind the section "Trimming segments and the word filter" of +[docs/vad-benchmarks.md](../../../docs/vad-benchmarks.md). No audio and no model files are +committed. `results/` holds the output of the run on that page. + +`CLI_OLD` is the build of master at the commit before this change (91b120b), `CLI_NEW` the build of +the change. Python packages: `numpy soundfile librosa datasets`. Models: the Ultra and Redux GGUFs +(`moondream/parakeet-ultra`, `moondream/parakeet-redux`, F16 and dequantized), a TDT 0.6B v3 F16 +GGUF, and a Silero GGUF (`scripts/convert_silero_vad_to_gguf.py`). + +``` +python3 fetch_data.py DATA # 3 TED-LIUM talks (not the first four) and LibriSpeech test-clean +python3 make_corpus.py DATA # inserts of synthetic noise, speech in noise, manifest.json +python3 run_all.py DATA OUT --old CLI_OLD --new CLI_NEW --ultra ultra-f16.gguf \ + --redux redux-deq-f16.gguf --v3 tdt-0.6b-v3-f16.gguf --silero silero-vad-f16.gguf -j 3 +python3 tables.py DATA OUT > results/tables.txt +rm -r DATA # audio +``` + +`fetch_data.py` streams the data; nothing else is downloaded. `make_corpus.py` uses fixed seeds, +so the files are the same on every run. `run_all.py` needs a few hours of CPU on a loaded machine +and resumes when you start it again. It is not a timing run: the machine was loaded (the load +average is in `results/tables.txt`). + +What the files are: + +- `talk_*`: three whole talks, with the reference text of the dataset. +- `ins__`: 90 s of a talk with 60 s of synthetic noise (white, pink, clicks, + music-like tones) inserted at a quiet point, at -20 or -35 dB against the speech. The noise block is + where every word is an invented word. +- `sn__`: six LibriSpeech utterances with gaps, clean, with white noise at 5 dB SNR or + with pink noise at 0 dB SNR. + +`tables.py` prints, for each detector (Ultra head, Redux head, v3 with Silero): the check that +`--vad-trim 0` gives the old output byte for byte, the WER with the old cuts, with trim 0.3 and with +trim 0.3 plus `--min-local-conf 0.5`, the seconds of the noise block that the decoder gets, and the +words it returns inside the block. diff --git a/scripts/vad_bench/decoder_guards/fetch_data.py b/scripts/vad_bench/decoder_guards/fetch_data.py new file mode 100644 index 0000000..fbdb676 --- /dev/null +++ b/scripts/vad_bench/decoder_guards/fetch_data.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""usage: fetch_data.py OUT [--skip K] [--talks N] [--libri N] + +Streams the data of the trim and word filter runs; nothing is kept but what is written to OUT: + OUT/talk_.wav, talk_.txt TED-LIUM long-form (test), talks under 1500 s, after the + first K talks (default 4: the talks that were used to tune + things earlier are skipped), N talks (default 3) + OUT/libri.npy, libri.json every 13th LibriSpeech test-clean utterance, first N (default 60) +Delete OUT after the runs: it is audio.""" +import argparse, io, json, os, re +import numpy as np, soundfile as sf, librosa +from datasets import Audio, load_dataset + +ap = argparse.ArgumentParser() +ap.add_argument("out"); ap.add_argument("--skip", type=int, default=4) +ap.add_argument("--talks", type=int, default=3); ap.add_argument("--libri", type=int, default=60) +a = ap.parse_args(); os.makedirs(a.out, exist_ok=True) + +ds = load_dataset("distil-whisper/tedlium-long-form", split="test", streaming=True).cast_column("audio", Audio(decode=False)) +kept = 0 +for i, ex in enumerate(ds): + if i < a.skip: continue + y, sr = sf.read(io.BytesIO(ex["audio"]["bytes"]), dtype="float32") + if y.ndim > 1: y = y.mean(axis=1) + if sr != 16000: y = librosa.resample(y, orig_sr=sr, target_sr=16000) + if len(y) / 16000 > 1500: continue + name = re.sub(r"[^A-Za-z0-9_-]", "", os.path.basename(ex["audio"]["path"] or f"talk{i}").rsplit(".", 1)[0]) or f"talk{i}" + sf.write(f"{a.out}/talk_{name}.wav", y, 16000, subtype="PCM_16") + open(f"{a.out}/talk_{name}.txt", "w").write(" ".join(re.sub(r"<[^>]*>", " ", ex["text"]).split()) + "\n") + print("talk", name, round(len(y) / 16000), flush=True); kept += 1 + if kept >= a.talks: break + +ds = load_dataset("openslr/librispeech_asr", "clean", split="test", streaming=True).cast_column("audio", Audio(decode=False)) +U, T = [], [] +for i, ex in enumerate(ds): + if i % 13: continue + y, sr = sf.read(io.BytesIO(ex["audio"]["bytes"]), dtype="int16"); assert sr == 16000 + U.append(y); T.append(ex["text"].lower()) + if len(U) >= a.libri: break +np.save(f"{a.out}/libri.npy", np.array(U, dtype=object), allow_pickle=True) +json.dump(T, open(f"{a.out}/libri.json", "w")) +print("libri", len(U), flush=True) diff --git a/scripts/vad_bench/decoder_guards/make_corpus.py b/scripts/vad_bench/decoder_guards/make_corpus.py new file mode 100644 index 0000000..7f8f8a8 --- /dev/null +++ b/scripts/vad_bench/decoder_guards/make_corpus.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""usage: make_corpus.py DATA + +Builds, from the output of fetch_data.py, the files of the runs (all 16 kHz mono, > 30 s so that the +segmenter cuts them) and DATA/manifest.json: + talk_* the talks as they are (kind "talk", reference text) + ins__ 90 s of a talk with 60 s of synthetic noise inserted at a quiet point + (kind "insert", "ins": [start, end] of the noise block). The noise level is + `rel` dB against the RMS of the speech. Noise: white, pink, clicks, music-like tones. + nz__ 30 s of noise alone (white, pink, clicks, music, hum, tone, sweep), no speech + (kind "noise"; shorter than the 30 s cap, so decoded whole) + sn__ 6 LibriSpeech utterances with gaps (kind "sinr"), clean or with white or pink noise at an SNR + (white5 = white noise at 5 dB, pink0 = pink noise at 0 dB) +Seeds are fixed, so the files are the same on every run.""" +import glob, json, os, sys +import numpy as np, soundfile as sf + +D = sys.argv[1]; SR = 16000 +man = [] + +def rms(x): return float(np.sqrt((x ** 2).mean()) + 1e-12) +def at_db(x, db): return x * (10 ** (db / 20) / rms(x)) +def pink(n, r): + X = np.fft.rfft(r.standard_normal(n)); f = np.arange(len(X)); f[0] = 1 + y = np.fft.irfft(X / np.sqrt(f), n); return y / y.std() +def clicks(n, r): + x = np.zeros(n); t = 0 + while t < n - 400: + k = np.arange(400); x[t:t + 400] += r.uniform(0.3, 1) * r.choice([-1, 1]) * np.exp(-k / 40) * np.sin(k * 0.8) + t += int(r.uniform(0.15, 0.5) * SR) + return x +def music(n, r): + t = np.arange(n) / SR; x = np.zeros(n) + for f0 in (220.0, 261.63, 329.63, 440.0): + f = f0 * (1 + 0.003 * np.sin(2 * np.pi * r.uniform(4, 6) * t + r.uniform(0, 6))) + ph = 2 * np.pi * np.cumsum(f) / SR + for h, a in ((1, 1.0), (2, 0.5), (3, 0.25)): x += a * np.sin(h * ph) + env = 0.6 + 0.4 * np.sin(2 * np.pi * 0.5 * t + r.uniform(0, 6)) ** 2 + return x * env +def hum(n, r): + t = np.arange(n) / SR; f = r.choice([50.0, 60.0]) + return sum(np.sin(2 * np.pi * f * h * t + r.uniform(0, 6)) / h for h in (1, 2, 3, 5)) +def tone(n, r): return np.sin(2 * np.pi * float(r.choice([440, 1000, 2000])) * np.arange(n) / SR) +def sweep(n, r): + t = np.arange(n) / SR; T = t[-1]; return np.sin(2 * np.pi * (50 * t + (7500 - 50) * t * t / (2 * T))) +GEN = {"white": lambda n, r: r.standard_normal(n), "pink": pink, "clicks": clicks, "music": music, + "hum": hum, "tone": tone, "sweep": sweep} + +# talks and inserts +for wav in sorted(glob.glob(f"{D}/talk_*.wav")): + name = os.path.basename(wav)[5:-4] + x, sr = sf.read(wav, dtype="float32"); assert sr == SR + man.append(dict(name=f"talk_{name}", kind="talk", talk=name, ref=open(wav[:-4] + ".txt").read().strip())) + ex = x[60 * SR:150 * SR]; db = 20 * np.log10(rms(ex)) + fr = len(ex) // 1600; e = 10 * np.log10((ex[:fr * 1600].reshape(fr, 1600) ** 2).mean(1) + 1e-12) + c = [(e[i:i + 10].mean(), i) for i in range(250, 650)]; _, i0 = min(c); cut = (i0 + 5) * 1600 + for ti, (tn, rel) in enumerate([("white", -20), ("pink", -20), ("clicks", -20), ("music", -20), ("white", -35), ("music", -35), + ("white", -5), ("pink", -5), ("clicks", -5), ("music", -5)]): + r = np.random.default_rng(1000 + 17 * ti + len(name)) + n = np.clip(at_db(GEN[tn](60 * SR, r), db + rel), -1, 1) + y = np.concatenate([ex[:cut], n, ex[cut:]]).astype(np.float32) + nm = f"ins_{name}_{tn}{rel}" + sf.write(f"{D}/{nm}.wav", y, SR, subtype="PCM_16") + man.append(dict(name=nm, kind="insert", talk=name, noise=tn, level=rel, ins=[cut / SR, cut / SR + 60])) + + # 30 s of noise alone, at the same levels against the speech (decoded whole, no cuts) + for ni, tn in enumerate(["white", "pink", "clicks", "music", "hum", "tone", "sweep"]): + for rel in (-35, -20, -5): + r = np.random.default_rng(5000 + 100 * ni + abs(rel) + len(name)) + nm = f"nz_{name}_{tn}{rel}" + sf.write(f"{D}/{nm}.wav", np.clip(at_db(GEN[tn](30 * SR, r), db + rel), -1, 1).astype(np.float32), SR, subtype="PCM_16") + man.append(dict(name=nm, kind="noise", talk=name, noise=tn, level=rel)) + +# speech in noise from LibriSpeech +U = [u.astype(np.float32) / 32768 for u in np.load(f"{D}/libri.npy", allow_pickle=True)] +T = json.load(open(f"{D}/libri.json")) +def trim(u): + fr = len(u) // 160; r = np.sqrt((u[:fr * 160].reshape(fr, 160) ** 2).mean(1)) + idx = np.where(r > r.max() * 0.01)[0]; return u[idx[0] * 160:(idx[-1] + 1) * 160] +for k in range(len(U) // 6): + ids = list(range(6 * k, 6 * k + 6)); r = np.random.default_rng(2000 + k) + parts, spans, t = [r.standard_normal(int(r.uniform(.5, 1.5) * SR)) * 1e-3], [], 0.0 + t = len(parts[0]) / SR + for j, i in enumerate(ids): + u = trim(U[i]); spans.append((t, t + len(u) / SR)); parts.append(u); t += len(u) / SR + g = r.standard_normal(int(r.uniform(.3, 2.0) * SR)) * 1e-3; parts.append(g); t += len(g) / SR + y = np.concatenate(parts).astype(np.float32) + sp = np.concatenate([y[int(s * SR):int(e * SR)] for s, e in spans]); P = (sp ** 2).mean() + for cond, kind, snr in (("clean", None, None), ("white5", "white", 5), ("pink0", "pink", 0)): + z = y + if kind: + rr = np.random.default_rng(3000 + k) + n = rr.standard_normal(len(y)) if kind == "white" else pink(len(y), rr) + z = np.clip(y + n * np.sqrt(P / 10 ** (snr / 10)), -1, 1).astype(np.float32) + nm = f"sn_{cond}_{k:02d}"; sf.write(f"{D}/{nm}.wav", z, SR, subtype="PCM_16") + man.append(dict(name=nm, kind="sinr", cond=cond, ref=" ".join(T[i] for i in ids))) +json.dump(man, open(f"{D}/manifest.json", "w")) +from collections import Counter +print(len(man), Counter(m["kind"] for m in man)) diff --git a/scripts/vad_bench/decoder_guards/results/tables.txt b/scripts/vad_bench/decoder_guards/results/tables.txt new file mode 100644 index 0000000..393985e --- /dev/null +++ b/scripts/vad_bench/decoder_guards/results/tables.txt @@ -0,0 +1,70 @@ +uptime when the tables were made: ['21.18', '43.26', '43.99'] (load average 1, 5, 15 min) + +== byte check: --vad-trim 0 against the old binary (JSON output, every file run both ways) +Ultra head 15/15 identical +Redux head 15/15 identical +v3 + Silero 15/15 identical + +== word error rate (%), words in the reference in brackets +-- talks: 3 files, 5627 words + old trim 0.3 +filter 0.5 trim - old filter - trim +Ultra head 3.68 3.68 3.68 +0.00 +0.00 +Redux head 4.39 4.51 - +0.12 - +v3 + Silero 3.45 3.45 3.45 +0.00 +0.00 +-- speech in noise: clean: 4 files, 429 words + old trim 0.3 +filter 0.5 trim - old filter - trim +Ultra head 2.10 1.86 1.86 -0.23 +0.00 +Redux head 3.26 3.03 - -0.23 - +v3 + Silero 1.86 1.86 1.86 +0.00 +0.00 +-- speech in noise: white5: 4 files, 429 words + old trim 0.3 +filter 0.5 trim - old filter - trim +Ultra head 4.90 4.43 4.43 -0.47 +0.00 +Redux head 7.23 6.76 - -0.47 - +v3 + Silero 5.59 5.13 5.13 -0.47 +0.00 +-- speech in noise: pink0: 4 files, 429 words + old trim 0.3 +filter 0.5 trim - old filter - trim +Ultra head 6.53 5.59 5.59 -0.93 +0.00 +Redux head 12.35 13.29 - +0.93 - +v3 + Silero 7.93 7.69 7.69 -0.23 +0.00 +-- talk DanielKahneman-merged +Ultra head 3.12 3.18 3.18 +Redux head 4.18 4.08 - +v3 + Silero 2.55 2.55 2.55 +-- talk EricMead_2009P_EricMead-merged +Ultra head 5.17 5.10 5.10 +Redux head 5.56 5.96 - +v3 + Silero 5.43 5.43 5.43 +-- talk GaryFlake-merged +Ultra head 3.18 3.09 3.09 +Redux head 3.36 3.72 - +v3 + Silero 3.18 3.18 3.18 + +== noise block of 60 s inside speech (insert files): seconds of the block that the decoder gets, and words in it +files: 30 (3 talks x white, pink, clicks, music at -20 and -5 dB, white and music at -35 dB) + sec old sec new words old words new +filter 0.5 files with words old/new/f05 +Ultra head 33.8 12.0 0 0 0 0/0/0 +Redux head 29.7 0.3 0 0 - 0/0/- +v3 + Silero 27.7 0.1 11 0 0 5/0/0 +per noise type, seconds of the block decoded (old -> new): +Ultra head clicks-20 26->0 clicks-5 48->7 music-20 13->0 music-35 24->0 music-5 13->0 pink-20 27->0 pink-5 60->52 white-20 47->1 white-35 21->0 white-5 60->58 +Redux head clicks-20 25->0 clicks-5 25->0 music-20 18->0 music-35 25->0 music-5 17->0 pink-20 37->0 pink-5 60->1 white-20 25->0 white-35 25->0 white-5 39->0 +v3 + Silero clicks-20 28->0 clicks-5 28->0 music-20 28->0 music-35 28->0 music-5 28->0 pink-20 28->0 pink-5 28->0 white-20 28->0 white-35 28->0 white-5 28->0 + +== what the filter (0.5) took out of the files with real speech +Ultra head words decoded 15375, hallucinated words in the noise blocks removed 0, other words removed 0 +v3 + Silero words decoded 15406, hallucinated words in the noise blocks removed 0, other words removed 3 + +== 30 s of noise alone, decoded whole (no VAD): every word is an invented word +files: 63 (3 talks x white, pink, clicks, music, hum, tone, sweep at -35, -20, -5 dB against the speech level) + files with words words +filter 0.5: files words +Ultra head 0 0 0 0 +Redux head 0 0 - - +v3 + Silero 1 1 0 0 + +== the filter alone (old cuts, --vad-trim 0, --min-local-conf 0.5) on the insert files whose noise block gave words +Ultra head 0 files: words in the block 0 -> 0; other words of those files 0, lost 0 +v3 + Silero 5 files: words in the block 11 -> 1; other words of those files 1484, lost 0 + +== higher thresholds on the speech in noise files (WER %, trim 0.3; words dropped of the words decoded) +Ultra head 12 files, 1287 reference words off: 3.96 (0 dropped) 0.5: 3.96 (0 dropped) 0.7: 3.96 (0 dropped) 0.9: 6.06 (31 dropped) +v3 + Silero 12 files, 1287 reference words off: 4.90 (0 dropped) 0.5: 4.90 (0 dropped) 0.7: 4.90 (0 dropped) 0.9: 16.08 (164 dropped) diff --git a/scripts/vad_bench/decoder_guards/run_all.py b/scripts/vad_bench/decoder_guards/run_all.py new file mode 100644 index 0000000..dd06289 --- /dev/null +++ b/scripts/vad_bench/decoder_guards/run_all.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""usage: run_all.py DATA OUT --old CLI_OLD --new CLI_NEW --ultra GGUF --redux GGUF --v3 GGUF --silero GGUF [-j N] + +Runs the CLI over the corpus of make_corpus.py and keeps every JSON output in OUT. CLI_OLD is +parakeet-cli built from the commit before the trim change, CLI_NEW the one with it. Per detector +(ultra head, redux head, v3 with Silero): + .old/.json transcribe --vad --json with CLI_OLD + .new/.json the same with CLI_NEW (trim 0.3, the default) + .new0/.json CLI_NEW with --vad-trim 0 (must equal .old byte for byte; a subset) + .f05/.json CLI_NEW with --min-local-conf 0.5 (ultra and v3 only) + .plain/.json, .plainf05/.json noise-only files, no VAD, without and with the filter + .old0f05/.json CLI_NEW with --vad-trim 0 --min-local-conf 0.5, only on the insert files in whose noise + block the old cuts gave words (the filter on its own, without the trim) + .f07, .f09/.json thresholds 0.7 and 0.9 on the speech in noise files (ultra and v3) + .segold/.json, .segnew/.json `vad --mode segments` of insert files (cuts) +Existing outputs are kept, so a run can be resumed. Not a timing run.""" +import argparse, json, os, subprocess, sys +from concurrent.futures import ThreadPoolExecutor + +ap = argparse.ArgumentParser() +ap.add_argument("data"); ap.add_argument("out") +for k in ("old", "new", "ultra", "redux", "v3", "silero"): ap.add_argument("--" + k, required=True) +ap.add_argument("--only", default="", help="regular expression: run only the files whose name matches") +ap.add_argument("-j", type=int, default=3); ap.add_argument("--sets", type=int, default=4, help="LibriSpeech sets per condition") +a = ap.parse_args() +man = [m for m in json.load(open(f"{a.data}/manifest.json")) + if m["kind"] != "sinr" or int(m["name"].rsplit("_", 1)[1]) < a.sets] +import re +if a.only: man = [m for m in man if re.search(a.only, m["name"])] +DET = {"ultra": (a.ultra, None), "redux": (a.redux, None), "v3": (a.v3, a.silero)} +jobs = [] +def add(out, cmd): + if not os.path.exists(out) or os.path.getsize(out) == 0: jobs.append((out, cmd)) +for det, (model, sil) in DET.items(): + vad = ["--vad"] + (["--vad-model", sil] if sil else []) + for mf in man: + wav = f"{a.data}/{mf['name']}.wav" + if mf["kind"] == "noise": # no VAD: the whole file is decoded, every word is an invented word + plain = ["transcribe", "--model", model, "--input", wav, "--json"] + for tag, extra in (("plain", []), ("plainf05", ["--min-local-conf", "0.5"])): + if tag == "plainf05" and det == "redux": continue + os.makedirs(f"{a.out}/{det}.{tag}", exist_ok=True) + add(f"{a.out}/{det}.{tag}/{mf['name']}.json", [a.new] + plain + extra) + continue + base = ["transcribe", "--model", model, "--input", wav, "--json"] + vad + def o(tag): os.makedirs(f"{a.out}/{det}.{tag}", exist_ok=True); return f"{a.out}/{det}.{tag}/{mf['name']}.json" + add(o("old"), [a.old] + base) + add(o("new"), [a.new] + base) + if mf["kind"] == "talk" or mf["name"].endswith(("_00", "_01")) or mf["name"].endswith(("white-20", "music-35")): + add(o("new0"), [a.new] + base + ["--vad-trim", "0"]) + if det != "redux": + add(o("f05"), [a.new] + base + ["--min-local-conf", "0.5"]) + if mf["kind"] == "insert": + seg = ["vad", "--model", sil or model, "--input", wav, "--mode", "segments"] + add(o("segold"), [a.old] + seg) + add(o("segnew"), [a.new] + seg) +print(len(jobs), "jobs", flush=True) +def run(j): + out, cmd = j + r = subprocess.run(cmd, capture_output=True, text=True) + if r.returncode == 0: open(out, "w").write(r.stdout) + else: print("FAILED", out, r.stderr[-300:], flush=True) + return out +with ThreadPoolExecutor(a.j) as ex: + for i, o in enumerate(ex.map(run, jobs)): + if i % 20 == 0: print(i, "/", len(jobs), flush=True) + +# Second pass: the filter alone, on the files where the old cuts gave invented words in the noise block. +jobs = [] +for det, (model, sil) in DET.items(): + vad = ["--vad"] + (["--vad-model", sil] if sil else []) + if det == "redux": continue + for mf in man: + if mf["kind"] != "insert": continue + f = f"{a.out}/{det}.old/{mf['name']}.json" + if not os.path.exists(f) or os.path.getsize(f) == 0: continue + lo, hi = mf["ins"] + if not any(lo + 0.5 <= w["start"] <= hi - 0.5 for w in json.load(open(f))["words"]): continue + os.makedirs(f"{a.out}/{det}.old0f05", exist_ok=True) + add(f"{a.out}/{det}.old0f05/{mf['name']}.json", + [a.new, "transcribe", "--model", model, "--input", f"{a.data}/{mf['name']}.wav", "--json"] + vad + ["--vad-trim", "0", "--min-local-conf", "0.5"]) +for det, (model, sil) in DET.items(): + if det == "redux": continue + vad = ["--vad"] + (["--vad-model", sil] if sil else []) + for mf in man: + if mf["kind"] != "sinr": continue + for tag, thr in (("f07", "0.7"), ("f09", "0.9")): + os.makedirs(f"{a.out}/{det}.{tag}", exist_ok=True) + add(f"{a.out}/{det}.{tag}/{mf['name']}.json", + [a.new, "transcribe", "--model", model, "--input", f"{a.data}/{mf['name']}.wav", "--json"] + vad + ["--min-local-conf", thr]) +print(len(jobs), "filter-only jobs", flush=True) +with ThreadPoolExecutor(a.j) as ex: + list(ex.map(run, jobs)) diff --git a/scripts/vad_bench/decoder_guards/tables.py b/scripts/vad_bench/decoder_guards/tables.py new file mode 100644 index 0000000..cb5928b --- /dev/null +++ b/scripts/vad_bench/decoder_guards/tables.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""usage: tables.py DATA OUT > results/tables.txt + +Reads the outputs of run_all.py and prints the tables of docs/vad-benchmarks.md (trim and word filter).""" +import json, os, re, sys, glob +import numpy as np + +D, O = sys.argv[1], sys.argv[2] +man = {m["name"]: m for m in json.load(open(f"{D}/manifest.json"))} +DETS = [("ultra", "Ultra head"), ("redux", "Redux head"), ("v3", "v3 + Silero")] + +def norm(s): return re.sub(r"[^a-z0-9' ]", " ", s.lower().replace("-", " ")).split() +def ed(a, b): + prev = list(range(len(b) + 1)) + for i, x in enumerate(a, 1): + cur = [i] + [0] * len(b) + for j, y in enumerate(b, 1): cur[j] = min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (x != y)) + prev = cur + return prev[-1] +def load(det, tag, name): + f = f"{O}/{det}.{tag}/{name}.json" + if not os.path.exists(f) or os.path.getsize(f) == 0: return None + return json.load(open(f)) +def wer(det, tag, names): + e = n = 0 + for nm in names: + j = load(det, tag, nm) + if j is None: return None + ref = norm(man[nm]["ref"]); e += ed(norm(j["text"]), ref); n += len(ref) + return 100.0 * e / n, n +def fmt(x): return "-" if x is None else f"{x[0]:.2f}" +def overlap(segs, a, b): return sum(max(0.0, min(s["end"], b) - max(s["start"], a)) for s in segs) + +print("uptime when the tables were made:", open("/proc/loadavg").read().split()[:3], "(load average 1, 5, 15 min)") +print("\n== byte check: --vad-trim 0 against the old binary (JSON output, every file run both ways)") +for det, lab in DETS: + eq = tot = 0 + for f in glob.glob(f"{O}/{det}.new0/*.json"): + nm = os.path.basename(f)[:-5]; o = f"{O}/{det}.old/{nm}.json" + if os.path.exists(o): tot += 1; eq += open(f).read() == open(o).read() + print(f"{lab:12s} {eq}/{tot} identical") + +print("\n== word error rate (%), words in the reference in brackets") +groups = [("talks", [n for n in man if man[n]["kind"] == "talk"])] +for c in ("clean", "white5", "pink0"): + groups.append((f"speech in noise: {c}", [n for n in man if man[n]["kind"] == "sinr" and man[n]["cond"] == c and load("ultra", "old", n) is not None])) +for gname, names in groups: + nref = sum(len(norm(man[n]["ref"])) for n in names) + print(f"-- {gname}: {len(names)} files, {nref} words") + print(f"{'':14s}{'old':>8s}{'trim 0.3':>10s}{'+filter 0.5':>13s}{'trim - old':>12s}{'filter - trim':>15s}") + for det, lab in DETS: + o, t, f = wer(det, "old", names), wer(det, "new", names), wer(det, "f05", names) + d1 = "-" if not (o and t) else f"{t[0] - o[0]:+.2f}" + d2 = "-" if not (t and f) else f"{f[0] - t[0]:+.2f}" + print(f"{lab:14s}{fmt(o):>8s}{fmt(t):>10s}{fmt(f):>13s}{d1:>12s}{d2:>15s}") +for tk in sorted({m["talk"] for m in man.values() if m["kind"] == "talk"}): + print(f"-- talk {tk}") + for det, lab in DETS: + o, t, f = wer(det, "old", [f"talk_{tk}"]), wer(det, "new", [f"talk_{tk}"]), wer(det, "f05", [f"talk_{tk}"]) + print(f"{lab:14s}{fmt(o):>8s}{fmt(t):>10s}{fmt(f):>13s}") + +print("\n== noise block of 60 s inside speech (insert files): seconds of the block that the decoder gets, and words in it") +ins = [n for n in man if man[n]["kind"] == "insert"] +print(f"files: {len(ins)} (3 talks x white, pink, clicks, music at -20 and -5 dB, white and music at -35 dB)") +print(f"{'':14s}{'sec old':>9s}{'sec new':>9s}{'words old':>11s}{'words new':>11s}{'+filter 0.5':>13s}{'files with words old/new/f05':>32s}") +def inblock(j, a, b): + return [w for w in j["words"] if a + 0.5 <= w["start"] <= b - 0.5] +for det, lab in DETS: + so, sn, wo, wn, wf, fo, fn, ff = [], [], 0, 0, 0, 0, 0, 0 + for nm in ins: + a, b = man[nm]["ins"] + jo, jn, jf = load(det, "old", nm), load(det, "new", nm), load(det, "f05", nm) + so_, sn_ = load(det, "segold", nm), load(det, "segnew", nm) + if so_ and sn_: so.append(overlap(so_["segments"], a, b)); sn.append(overlap(sn_["segments"], a, b)) + if jo: k = len(inblock(jo, a, b)); wo += k; fo += k > 0 + if jn: k = len(inblock(jn, a, b)); wn += k; fn += k > 0 + if jf: k = len(inblock(jf, a, b)); wf += k; ff += k > 0 + print(f"{lab:14s}{np.mean(so):9.1f}{np.mean(sn):9.1f}{wo:11d}{wn:11d}{(str(wf) if det != 'redux' else '-'):>13s}{f'{fo}/{fn}/{ff}' if det != 'redux' else f'{fo}/{fn}/-':>32s}") +print("per noise type, seconds of the block decoded (old -> new):") +for det, lab in DETS: + row = {} + for nm in ins: + a, b = man[nm]["ins"]; so_, sn_ = load(det, "segold", nm), load(det, "segnew", nm) + if so_ and sn_: row.setdefault(f"{man[nm]['noise']}{man[nm]['level']}", []).append((overlap(so_["segments"], a, b), overlap(sn_["segments"], a, b))) + print(f"{lab:14s}" + " ".join(f"{k} {np.mean([x[0] for x in v]):.0f}->{np.mean([x[1] for x in v]):.0f}" for k, v in sorted(row.items()))) + +print("\n== what the filter (0.5) took out of the files with real speech") +for det, lab in DETS: + if det == "redux": continue + dropped = 0; words = 0; hall = 0 + for nm, m in man.items(): + j, t = load(det, "f05", nm), load(det, "new", nm) + if j is None or t is None: continue + d = j.get("guard", {}).get("dropped_words", 0) + if m["kind"] == "insert": + a, b = m["ins"]; hall += len(inblock(t, a, b)) - len(inblock(j, a, b)) + # words outside the block that went with it + dropped += d - (len(inblock(t, a, b)) - len(inblock(j, a, b))) + else: dropped += d + words += len(t["words"]) + print(f"{lab:12s} words decoded {words}, hallucinated words in the noise blocks removed {hall}, other words removed {dropped}") + +print("\n== 30 s of noise alone, decoded whole (no VAD): every word is an invented word") +nz = [n for n in man if man[n]["kind"] == "noise"] +print(f"files: {len(nz)} (3 talks x white, pink, clicks, music, hum, tone, sweep at -35, -20, -5 dB against the speech level)") +print(f"{'':14s}{'files with words':>18s}{'words':>8s}{'+filter 0.5: files':>20s}{'words':>8s}") +for det, lab in DETS: + fw = w = ffw = fw5 = 0 + for nm in nz: + j, f = load(det, "plain", nm), load(det, "plainf05", nm) + if j is None: continue + k = len(j["words"]); w += k; fw += k > 0 + if f is not None: fw5 += len(f["words"]); ffw += len(f["words"]) > 0 + print(f"{lab:14s}{fw:18d}{w:8d}{(str(ffw) if det != 'redux' else '-'):>20s}{(str(fw5) if det != 'redux' else '-'):>8s}") + +print("\n== the filter alone (old cuts, --vad-trim 0, --min-local-conf 0.5) on the insert files whose noise block gave words") +for det, lab in DETS: + if det == "redux": continue + files = hall = left = lost = real = 0 + for nm in ins: + j, f = load(det, "old", nm), load(det, "old0f05", nm) + if j is None or f is None: continue + a, b = man[nm]["ins"]; files += 1 + k0, k1 = len(inblock(j, a, b)), len(inblock(f, a, b)) + hall += k0; left += k1 + lost += len(j["words"]) - len(f["words"]) - (k0 - k1); real += len(j["words"]) - k0 + print(f"{lab:12s} {files} files: words in the block {hall} -> {left}; other words of those files {real}, lost {lost}") + +print("\n== higher thresholds on the speech in noise files (WER %, trim 0.3; words dropped of the words decoded)") +sn = [n for n in man if man[n]["kind"] == "sinr" and load("ultra", "f05", n) is not None] +for det, lab in DETS: + if det == "redux": continue + row = [] + for tag, t in (("new", "off"), ("f05", "0.5"), ("f07", "0.7"), ("f09", "0.9")): + w = wer(det, tag, [n for n in sn if load(det, tag, n) is not None]) + dr = sum(load(det, tag, n).get("guard", {}).get("dropped_words", 0) for n in sn if load(det, tag, n) is not None) + row.append(f"{t}: {fmt(w)} ({dr} dropped)") + print(f"{lab:12s} {len(sn)} files, {sum(len(norm(man[n]['ref'])) for n in sn)} reference words " + " ".join(row))