feat(cursor): play a click sound on every recorded click

THIRD-PARTY-NOTICES.md

@@ -257,3 +257,17 @@ and are covered by [LICENSE](LICENSE).
 
 To report an omission or request source for anything bundled here, open an issue
 at <https://github.com/getopenscreen/openscreen/issues>.
+---
+
+## Kenney — UI sound effects (mouse click sound)
+
+- **Files**: `public/sounds/click-down.wav` (from `click3.wav`) and
+  `public/sounds/click-up.wav` (from `switch12.wav`) of the "UI Audio" set. Both were
+  trimmed of their leading silence and levelled to a 0.7 peak, so a hit lands on the
+  click and the two samples play at matching loudness without the mixer measuring them.
+- **Used by**: the "Mouse clicks" editor toggle, which lays these two hits along the
+  click timestamps recorded with the take.
+- **License**: **CC0 1.0 Universal (Public Domain Dedication)**,
+  <https://creativecommons.org/publicdomain/zero/1.0/>. The pack ships a readme stating
+  personal and commercial use is permitted and that credit is optional; upstream:
+  <https://kenney.nl/assets/ui-audio>.

crates/compositor/src/audio.rs

@@ -5,7 +5,9 @@
 use crate::ffi::*;
 
 use crate::regions::SpeedSegment;
-use crate::scene::{SceneAudio, SceneAudioTrack, SceneAudioTrackKind};
+use crate::scene::{
+    SceneAudio, SceneAudioTrack, SceneAudioTrackKind, SceneClickHit, SceneClickSound,
+};
 use anyhow::{bail, Result};
 use std::collections::{HashMap, VecDeque};
 use std::f32::consts::PI;
@@ -1732,6 +1734,67 @@ fn overlay_external_track(programme: &mut PlanarPcm, track: &SceneAudioTrack, du
     );
 }
 
+/// How much of a hit file gets decoded. The bundled samples are a few hundred milliseconds;
+/// the window is what keeps a wrong path from buffering an hour of "click".
+/// ponytail: fixed window, read the file's own duration if longer hits are ever shipped.
+const CLICK_HIT_WINDOW_SEC: f64 = 1.0;
+
+/// Sum the take's click hits over the assembled programme: two decodes for the whole take,
+/// then one add per hit — unlike an imported track, which decodes once per placement.
+///
+/// The hits arrive already placed on the programme by the app, through the projection it lays
+/// imported audio tracks out with, so speeding a section up cannot leave the sound behind.
+pub fn mix_click_hits(mut programme: PlanarPcm, clicks: &SceneClickSound) -> PlanarPcm {
+    if clicks.hits.is_empty()
+        || clicks.down_path.is_empty()
+        || clicks.up_path.is_empty()
+        || programme.first().map_or(true, Vec::is_empty)
+    {
+        return programme;
+    }
+    let (down, up) = (
+        decode_click_hit(&clicks.down_path),
+        decode_click_hit(&clicks.up_path),
+    );
+    if down.is_empty() || up.is_empty() {
+        eprintln!("[audio] click hits undecodable; export keeps the picture only");
+        return programme;
+    }
+    overlay_click_hits(&mut programme, &down, &up, &clicks.hits);
+    programme
+}
+
+fn decode_click_hit(path: &str) -> PlanarPcm {
+    decode_clip_audio(path, 0.0, CLICK_HIT_WINDOW_SEC)
+        .ok()
+        .flatten()
+        .unwrap_or_default()
+}
+
+/// Placement and level, with no ffmpeg in sight. `overlay_track_pcm` truncates at the
+/// programme's end, so a hit the edit pushed past it is simply not heard.
+fn overlay_click_hits(
+    programme: &mut PlanarPcm,
+    down: &PlanarPcm,
+    up: &PlanarPcm,
+    hits: &[SceneClickHit],
+) {
+    for hit in hits {
+        let offset = (hit.time_sec.max(0.0) * AUDIO_OUTPUT_SAMPLE_RATE as f64).round() as usize;
+        let gain = hit.gain.clamp(0.0, 4.0);
+        overlay_track_pcm(
+            programme,
+            if hit.release { up } else { down },
+            offset,
+            gain,
+            0.0,
+            0.0,
+            0,
+            &[],
+        );
+    }
+}
+
 /// How far a music bed dips while someone speaks.
 pub const DUCK_DEPTH_DB: f32 = -10.0;
 /// A 10 ms stretch of the voice counts as speech above this RMS level. The voice is already
@@ -2268,6 +2331,66 @@ mod tests {
         assert_eq!(programme[0], vec![0.3, 0.3]);
     }
 
+    fn hit(time_sec: f64, gain: f32, release: bool) -> SceneClickHit {
+        SceneClickHit { time_sec, gain, release }
+    }
+
+    #[test]
+    fn click_hits_sum_at_their_time_and_pick_their_sample() {
+        // down is 1.0 and up is 0.5, so the level heard at each offset says which sample played.
+        let (down, up) = (planar(&[1.0]), planar(&[0.5]));
+        let tick = 1.0 / AUDIO_OUTPUT_SAMPLE_RATE as f64;
+        let mut programme = planar(&[0.0, 0.0, 0.0]);
+        overlay_click_hits(
+            &mut programme,
+            &down,
+            &up,
+            &[
+                hit(0.0, 1.0, false),
+                hit(tick, 1.0, true),
+                hit(2.0 * tick, 0.5, false),
+            ],
+        );
+        assert_eq!(programme[0], vec![1.0, 0.5, 0.5]);
+    }
+
+    #[test]
+    fn a_click_hit_the_edit_pushed_past_the_programme_is_not_heard() {
+        let mut programme = planar(&[0.25, 0.25]);
+        overlay_click_hits(
+            &mut programme,
+            &planar(&[1.0]),
+            &planar(&[1.0]),
+            &[hit(5.0, 1.0, false)],
+        );
+        assert_eq!(programme[0], vec![0.25, 0.25]);
+    }
+
+    #[test]
+    fn click_hits_are_decoded_from_files_and_summed_at_their_time() {
+        // The whole point of the scene carrying hits instead of a baked bed: the samples live in
+        // their own files and are read here, so this walks the same decode a real track does.
+        let dir = std::env::temp_dir();
+        let down = dir.join(format!("openscreen-click-down-{}.wav", std::process::id()));
+        let up = dir.join(format!("openscreen-click-up-{}.wav", std::process::id()));
+        write_wav(&down, &planar(&[0.5, 0.5]));
+        write_wav(&up, &planar(&[0.25, 0.25]));
+        let clicks = SceneClickSound {
+            down_path: down.to_str().unwrap().into(),
+            up_path: up.to_str().unwrap().into(),
+            hits: vec![hit(0.0, 1.0, false), hit(1.0 / 48_000.0, 1.0, true)],
+        };
+        let out = mix_click_hits(planar(&[0.0; 4]), &clicks);
+        let _ = std::fs::remove_file(&down);
+        let _ = std::fs::remove_file(&up);
+        // Each sample is two frames long: the press covers 0..1 and the hit placed one frame in
+        // covers 1..2, so the seam carries both and the programme is untouched past them.
+        assert_eq!(
+            out[0].iter().map(|v| (v * 100.0).round() / 100.0).collect::<Vec<_>>(),
+            vec![0.5, 0.75, 0.25, 0.0]
+        );
+    }
+
     #[test]
     fn mix_external_tracks_skips_empty_windows() {
         let programme = planar(&[0.4, 0.4]);

crates/compositor/src/pipeline_linux.rs

@@ -27,7 +27,8 @@ use std::ffi::CString;
 use std::ptr;
 
 use crate::audio::{
-    assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_external_tracks,
+    assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_click_hits,
+    mix_external_tracks,
     AacEncoder, PlanarPcm,
 };
 use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs};
@@ -1019,6 +1020,11 @@ pub fn run_composited_multi(
         .as_ref()
         .map(|scene| scene.audio_tracks.clone())
         .unwrap_or_default();
+    // Click hits (the "mouse clicks" toggle), laid on the programme by the app.
+    let click_sound = scene
+        .as_ref()
+        .map(|scene| scene.click_sound.clone())
+        .unwrap_or_default();
     // Ring de staging a 2 : l'export ne veut que du debit, une frame de latence
     // ne se voit pas dans un fichier. Voir `Compositor::set_readback_depth` pour
     // la raison pour laquelle la preview, elle, reste a 1.
@@ -1208,7 +1214,10 @@ pub fn run_composited_multi(
         let octx = mux.octx;
         mux.aac.encode(
             &finish_audio(
-                mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks),
+                mix_click_hits(
+                    mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks),
+                    &click_sound,
+                ),
                 audio_settings,
             ),
             octx,

crates/compositor/src/pipeline_macos.rs

@@ -30,7 +30,8 @@
 //! décodeurs, symétrique.
 
 use crate::audio::{
-    assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_external_tracks,
+    assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_click_hits,
+    mix_external_tracks,
     AacEncoder, PlanarPcm,
 };
 use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs};
@@ -1231,6 +1232,11 @@ pub fn run_composited_multi(
         .as_ref()
         .map(|scene| scene.audio_tracks.clone())
         .unwrap_or_default();
+    // Click hits (the "mouse clicks" toggle), laid on the programme by the app.
+    let click_sound = scene
+        .as_ref()
+        .map(|scene| scene.click_sound.clone())
+        .unwrap_or_default();
     frames = unsafe {
         crate::timeline_walk::walk_composited_timeline(
             clips,
@@ -1307,7 +1313,10 @@ pub fn run_composited_multi(
         let plan = build_audio_concat_plan(&clip_frame_counts, &declared_audio, out_fps as f64);
         audio_encoder.encode(
             &finish_audio(
-                mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks),
+                mix_click_hits(
+                    mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks),
+                    &click_sound,
+                ),
                 audio_settings,
             ),
             octx,

crates/compositor/src/pipeline_windows.rs

@@ -3,7 +3,8 @@
 //! tout le run, deux lectures seulement. Rien dans la boucle ne peut fausser le fps.
 
 use crate::audio::{
-    assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_external_tracks,
+    assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_click_hits,
+    mix_external_tracks,
     AacEncoder, PlanarPcm,
 };
 use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs};
@@ -1668,6 +1669,11 @@ unsafe fn run_multi_inner(
         .as_ref()
         .map(|scene| scene.audio_tracks.clone())
         .unwrap_or_default();
+    // Click hits (the "mouse clicks" toggle), laid on the programme by the app.
+    let click_sound = scene
+        .as_ref()
+        .map(|scene| scene.click_sound.clone())
+        .unwrap_or_default();
 
     // ---- encodeur (choisi à l'exécution, cf. ExportCodec::candidates) + mux ----
     // Backend CPU : pas de pool D3D11 du tout. `av_hwdevice_ctx_init(D3D11VA)` échoue sur
@@ -1804,7 +1810,10 @@ unsafe fn run_multi_inner(
         out_fps as f64,
     );
     let assembled_audio = finish_audio(
-        mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &audio_plan), &audio_tracks),
+        mix_click_hits(
+            mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &audio_plan), &audio_tracks),
+            &click_sound,
+        ),
         audio_settings,
     );
     audio_encoder.encode(&assembled_audio, octx)?;

crates/compositor/src/scene.rs

@@ -666,6 +666,43 @@ pub enum SceneAudioTrackKind {
     Voiceover,
 }
 
+/// One click to be heard: `time_sec` is its place on the OUTPUT programme, `gain` the linear
+/// level of the hit (side clicks a little quieter than the left), `release` which of the two
+/// samples to play.
+#[derive(Debug, Clone, Copy, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct SceneClickHit {
+    pub time_sec: f64,
+    #[serde(default = "default_one")]
+    pub gain: f32,
+    #[serde(default)]
+    pub release: bool,
+}
+
+/// The "mouse clicks" toggle: the recorded click times, already projected onto the programme,
+/// plus the two hit samples to play them with.
+///
+/// Deliberately not an imported audio track. A bed baked into one file cannot follow an edit —
+/// `SceneAudioTrack`'s content is placed, never stretched, so every hit after a speed region
+/// drifts — while hits laid out here are positioned by the app's own projection, the same one
+/// imported audio tracks are placed with, so they cannot disagree with the picture.
+#[derive(Debug, Clone, Default, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct SceneClickSound {
+    /// Absolute paths to the press and release samples, 48 kHz-friendly WAV files. `default` for
+    /// the usual reason: a half-written block must cost the sound, never the whole scene parse.
+    #[serde(default)]
+    pub down_path: String,
+    #[serde(default)]
+    pub up_path: String,
+    #[serde(default)]
+    pub hits: Vec<SceneClickHit>,
+}
+
+fn default_one() -> f32 {
+    1.0
+}
+
 #[derive(Debug, Clone, Copy, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct SceneOutput {
@@ -738,6 +775,10 @@ pub struct Scene {
     /// absent from every scene written before this, and from a project with none.
     #[serde(default)]
     pub audio_tracks: Vec<SceneAudioTrack>,
+    /// Click hits for the "mouse clicks" toggle. `#[serde(default)]`: an empty `hits` is the
+    /// same as the field being absent, which is every scene written before this.
+    #[serde(default)]
+    pub click_sound: SceneClickSound,
     /// Crop écran par clip, dans le même ordre que `clips` (`cropByClip` côté TS).
     #[serde(default)]
     pub crop_by_clip: Vec<Option<SceneCrop>>,
@@ -912,6 +953,24 @@ mod tests {
         assert_eq!(effects(r#","frame":"laptop""#).frame.theme_override(), None);
     }
 
+    #[test]
+    fn a_click_sound_payload_reads_the_apps_camel_case_names() {
+        let clicks: SceneClickSound = serde_json::from_str(
+            r#"{"downPath":"/u/click-down.wav","upPath":"/u/click-up.wav","hits":[
+                {"timeSec":2.5,"gain":0.7,"release":true},{"timeSec":4.0}]}"#,
+        )
+        .unwrap();
+        assert_eq!(clicks.down_path, "/u/click-down.wav");
+        assert_eq!(clicks.hits.len(), 2);
+        assert_eq!(clicks.hits[0].time_sec, 2.5);
+        assert!(clicks.hits[0].release);
+        // A hit that carries no level is a full-strength press.
+        assert_eq!(clicks.hits[1].gain, 1.0);
+        // And a scene from before the field reads as no hits, not as a parse failure.
+        assert!(SceneClickSound::default().hits.is_empty());
+        assert!(serde_json::from_str::<SceneClickSound>("{}").unwrap().hits.is_empty());
+    }
+
     #[test]
     fn parses_color_and_image_backgrounds() {
         let color = r##"{"clips":[],"layout":{"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false},"effects":{"padding":0,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0},"background":{"kind":"color","color":"#123456"},"zoomRegions":[],"cursor":{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"},"cropByClip":[],"output":{"width":1280,"height":720,"fps":30}}"##;

electron/ai-edition/style-preset-service.test.ts

@@ -34,6 +34,8 @@ const APPEARANCE: StylePresetAppearance = {
 		model3d: false,
 		asArrow: [],
 		clickImpact: false,
+		clickSound: false,
+		clickSoundGainDb: 0,
 	},
 	cursorShow: true,
 	cursorAutoHide: false,

electron/electron-env.d.ts

@@ -338,6 +338,17 @@ interface Window {
 			message?: string;
 			error?: string;
 		}>;
+		// Stage one of the two bundled click-sound samples where the compositor can read it
+		// by path. Only the two names the app ships are accepted, so nothing is sanitised here.
+		stageClickSoundHit: (
+			name: string,
+			data: ArrayBuffer,
+		) => Promise<{
+			success: boolean;
+			path?: string;
+			message?: string;
+			error?: string;
+		}>;
 		setCurrentVideoPath: (path: string) => Promise<{ success: boolean }>;
 		setCurrentRecordingSession: (
 			session: import("../src/lib/recordingSession").RecordingSession | null,