Add opt-in audio description for videos

CONTRIBUTING.md

@@ -191,6 +191,7 @@ published manifests/media and send playback events to Core.
 Technical reference, as needed:
 
 - [Flow engine](docs/flow_engine.md): templates, execution, retries and debugging.
+- [Audio description](docs/audio_description.md): steps, outputs, providers and host policy.
 - [Prepared analytics](docs/analytics_sessions.md): setup and data lifecycle.
 - [Webhook delivery](docs/webhook_delivery_outbox.md): signed lifecycle events.
 - [Encoding boosters](deploy/encoding-booster/README.md): optional remote compute;

README.md

@@ -20,8 +20,8 @@ and publish videos — with your media in S3-compatible storage that you control
 
 - **Manage:** organize videos into folders, invite your team, and replace a
   video without changing its public embed.
-- **Publish:** resumable uploads, adaptive HLS, posters, subtitles, and
-  customizable player settings.
+- **Publish:** resumable uploads, adaptive HLS, posters, subtitles,
+  [audio description](docs/audio_description.md), and customizable player settings.
 - **Integrate:** embed with web components, iframe, React, or Vue; automate
   with the video API and signed webhooks.
 - **Understand:** see views, drop-off, and viewing sources in your dashboard,

assets/js/app.js

@@ -28,6 +28,7 @@ import topbar from "../vendor/topbar"
 import "@lottiefiles/lottie-player"
 import { ChartHook } from "./hooks/chart"
 import { cdn } from "./upload"
+import { FileMenuHook } from "./hooks/file_menu"
 
 let selectedEmbed = null
 
@@ -717,6 +718,7 @@ const liveSocket = new LiveSocket("/live", Socket, {
     draggable: DraggableEmbedHook,
     thumbnail_preview: ThumbnailPreviewHook,
     auto_open_file_picker: AutoOpenFilePickerHook,
+    file_menu: FileMenuHook,
     preserve_scroll: PreserveScrollHook,
   },
   uploaders: { cdn },

assets/js/hooks/file_menu.js

@@ -0,0 +1,89 @@
+export const FileMenuHook = {
+  mounted() {
+    this.onClick = (event) => {
+      const trigger = event.target.closest("[data-file-menu-trigger='true']")
+      if (trigger && this.el.contains(trigger)) {
+        this.isOpen ? this.closeMenu() : this.openMenu(event.detail === 0)
+      } else if (event.target.closest("[role='menuitem']")) {
+        this.closeMenu()
+      }
+    }
+    this.onOutside = (event) => {
+      if (!this.el.contains(event.target)) this.closeMenu()
+    }
+    this.onKey = (event) => {
+      if (!this.isOpen) return
+      if (event.key === "Escape") {
+        event.preventDefault()
+        this.closeMenu(true)
+      } else if (event.key === "Tab") {
+        this.closeMenu()
+      } else if (["ArrowDown", "ArrowUp", "Home", "End"].includes(event.key)) {
+        event.preventDefault()
+        const items = [...this.panel.querySelectorAll("[role='menuitem']:not(:disabled)")]
+        const current = items.indexOf(document.activeElement)
+        const next = event.key === "Home" ? 0 : event.key === "End" ? items.length - 1 :
+          current < 0 ? (event.key === "ArrowDown" ? 0 : items.length - 1) :
+          (current + (event.key === "ArrowDown" ? 1 : -1) + items.length) % items.length
+        items[next]?.focus()
+      }
+    }
+    this.onScroll = (event) => {
+      if (this.isOpen && !this.panel?.contains(event.target)) this.positionMenu()
+    }
+    this.onResize = () => this.closeMenu()
+    this.el.addEventListener("click", this.onClick)
+    this.el.addEventListener("keydown", this.onKey)
+    document.addEventListener("pointerdown", this.onOutside)
+    document.addEventListener("scroll", this.onScroll, true)
+    window.addEventListener("resize", this.onResize)
+  },
+
+  updated() {
+    this.closeMenu()
+  },
+
+  openMenu(focusFirst = false) {
+    this.trigger = this.el.querySelector("[data-file-menu-trigger='true']")
+    this.panel = this.el.querySelector("[data-file-menu-panel]")
+    if (!this.trigger || !this.panel) return
+
+    // Keep the menu clear of the narrow sidebar's scroll clipping and viewport edges.
+    this.panel.style.display = "block"
+    this.positionMenu()
+    this.trigger.setAttribute("aria-expanded", "true")
+    this.isOpen = true
+    if (focusFirst) this.panel.querySelector("[role='menuitem']:not(:disabled)")?.focus({ preventScroll: true })
+  },
+
+  positionMenu() {
+    const anchor = this.trigger.getBoundingClientRect()
+    if (anchor.bottom <= 0 || anchor.top >= window.innerHeight) {
+      this.closeMenu()
+      return
+    }
+    const menu = this.panel.getBoundingClientRect()
+    const inset = 8
+    const rowRight = this.el.parentElement.getBoundingClientRect().right
+    const left = Math.max(inset, Math.min(rowRight - menu.width, window.innerWidth - menu.width - inset))
+    const top = anchor.bottom + menu.height + inset <= window.innerHeight ?
+      anchor.bottom + inset : Math.max(inset, anchor.top - menu.height - inset)
+    this.panel.style.left = `${left}px`
+    this.panel.style.top = `${top}px`
+  },
+
+  closeMenu(returnFocus = false) {
+    if (this.panel) this.panel.style.display = "none"
+    this.trigger?.setAttribute("aria-expanded", "false")
+    this.isOpen = false
+    if (returnFocus) this.trigger?.focus({ preventScroll: true })
+  },
+
+  destroyed() {
+    this.el.removeEventListener("click", this.onClick)
+    this.el.removeEventListener("keydown", this.onKey)
+    document.removeEventListener("pointerdown", this.onOutside)
+    document.removeEventListener("scroll", this.onScroll, true)
+    window.removeEventListener("resize", this.onResize)
+  },
+}

docs/analytics_sessions.md

@@ -93,7 +93,12 @@ mix analytics.sessions.backfill --materialize-inventory --materialize-session-re
 Source deletion, copying or restoration must invalidate affected prepared data
 with `Sessions.invalidate/1` under the caller's existing authorization, then
 rebuild it. Do not reuse a retired space identifier before invalidation succeeds.
-Normal event arrival needs no explicit invalidation.
+Invalidation atomically hides the published generations in PostgreSQL and records
+them for cleanup; it does not wait for ClickHouse deletion. New readers cannot
+select the old generations, while reads already in progress retain the reader
+grace period. The background processor rebuilds from current source events, and
+cleanup reclaims retired generations independently. Normal event arrival needs
+no explicit invalidation.
 
 ## Query semantics
 

docs/audio_description.md

@@ -0,0 +1,163 @@
+# Audio description
+
+Audio description makes a video followable for blind and low-vision viewers: a
+voice describes what happens on screen. A space opts in under **Settings →
+Features → Audio description**. From then on, every new upload and reprocessing
+run of a video of at most 10 minutes gets one description in its detected
+spoken language (English when no language is known):
+
+- `audio_description_<lang>.mp3`: a full mix of the video's own audio with the
+  spoken descriptions. It is also an HLS alternate audio rendition, so players
+  can switch to it.
+- `descriptions_<lang>.vtt`: a WebVTT with `Kind: descriptions`, containing exactly
+  the spoken descriptions, timed to when they are spoken.
+
+Existing videos only get audio descriptions when they are processed again.
+
+The flow input can set `audio_description.fallback_language` (default `en`).
+A `fallback_language` step parameter overrides that default. These values are
+used only when the original language is unknown; hosts can snapshot a space's
+preferred fallback in the flow input without coupling the pipeline to space settings.
+
+## How descriptions are placed
+
+- **Original language.** The picture is described directly in the language
+  detected by transcription. If no language is known, English is used. No second
+  language or translation is generated by default.
+- **In the pauses, otherwise ducked.** Where speech is, is measured with
+  speech-to-text (the `ai.transcribe_audio` segments, or a transcription made for
+  this purpose), never guessed by the description model. A description that fits a
+  pause near its moment (up to 1 s earlier or 2 s later) plays there and leaves the
+  original audio untouched. When none fits, it plays at its moment and the original
+  audio fades to 20% underneath it (0.3 s fades). Descriptions are not shortened to
+  make room.
+- **In sync with the picture.** Cuts are detected with FFmpeg `scdet` and grouped
+  into scenes of at least 3 s, so a fast montage is still described. A description
+  starts while its scene is on screen and at least half of it is spoken during the
+  scene; a pause after the cut never counts as room for it. One that can no longer
+  start in its scene, because earlier descriptions ran long, is left out.
+- **Labelled stills.** The image model gets one 360p still per ~3 s per shot
+  (at least one per cut), labelled `S<scene> at <time>`, in batches of 12, with the
+  scenes, speech and numbered pauses (each with a word budget) as context.
+
+## Flow steps
+
+The publishing presets (`publish_default`, `publish_local`, `publish_remote`,
+`publish_remote_local`) contain these optional background steps. Each one skips
+itself (`status: "skipped"`, `reason: "not_requested"`) unless the run input has
+`audio_description`, so disabled spaces do not call the AI providers. Nothing in the publish
+path waits for them; when a mix step finishes, Core stores the result, rebuilds the
+HLS master playlist and republishes `manifest.json`.
+
+| Step | Type | Runs on | Does |
+| --- | --- | --- | --- |
+| `detect_scenes` | `media.detect_scenes` | FLAME (media pool) | Cuts, scenes and stills in one decode of the canonical source |
+| `describe_video` | `ai.describe_video` | inline | Original-language cues |
+| `speak_descriptions_original` | `ai.speak_descriptions` | inline | Text-to-speech per cue; existing clips are reused on retry |
+| `mix_audio_description_original` | `media.mix_audio_description` | FLAME (media pool) | Timeline, ducked mix, VTT and the HLS audio rendition |
+
+Run input:
+
+```json
+{"audio_description": {"languages": ["original"], "fallback_language": "en"}}
+```
+
+Uploads and the public `input_url` API add this when the space has the feature on
+and the host allows it at that moment (see below). Existing runs keep their
+original versioned flow definition; newly started runs use the single-language preset.
+Videos longer than `MAVE_AUDIO_DESCRIPTION_MAX_SECONDS` (default `600`) are
+skipped with `reason: "video_too_long"`, `video_duration_seconds` and
+`max_duration_seconds`; videos without a video stream with `no_video`.
+
+Transient provider and storage errors (timeouts, 408/429/5xx) are retried with the
+flow engine's retry budget. Other failures make the step `unavailable`; they never
+block publication. Working objects (stills, cues, speech clips) are private and
+live under `<embed>/vN/audio_description/`.
+
+## Outputs
+
+Files sit next to the other media of the video version (`<embed>/vN/`):
+`audio_description_<lang>.mp3`, `audio_description_<lang>_hls/playlist.m3u8` and
+`descriptions_<lang>.vtt`. Generated rows are identified by video version and
+generation language; uploaded tracks may share the same language.
+
+The HLS master playlist lists each MP3 in the `audio` group:
+
+```text
+#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="Dutch (audio description)",LANGUAGE="nl",DEFAULT=NO,AUTOSELECT=YES,CHARACTERISTICS="public.accessibility.describes-video",CHANNELS="2",URI="audio_description_nl_hls/playlist.m3u8"
+```
+
+`manifest.json` has a separate key, so players that don't know it are unaffected
+(the existing `audio_tracks` and `subtitles` don't include audio descriptions):
+
+```json
+"audio_descriptions": [
+  {
+    "language": "nl",
+    "label": "Dutch (audio description)",
+    "audio": {"path": "https://…/audio_description_nl.mp3", "hls_playlist": "audio_description_nl_hls/playlist.m3u8", "file_size": 123456},
+    "vtt": {"path": "https://…/descriptions_nl.vtt", "file_size": 1234}
+  }
+]
+```
+
+The key is omitted when a video has none. The detailed video API payload has the
+same `audio_descriptions` list. The dashboard video page lists the available
+languages and HLS/MP3/VTT formats under **Audio descriptions** in Settings and
+**Audio** in Info, where descriptions are marked `(AD)`.
+
+The file menu on audio tracks, audio descriptions and subtitles provides file
+downloads and replacement. Removing a track asks for confirmation.
+
+Under **Settings → Audio descriptions**, tracks can be uploaded, renamed,
+retagged with a language, replaced, or removed. Upload a complete MP3 mix with
+both the original audio and spoken descriptions; the player switches tracks
+rather than mixing them. Uploaded tracks can share a language. A manually
+replaced track has no generated VTT, since the old text may not match the new
+audio. The replacement menu also supports uploading or replacing the associated
+VTT. The generated pipeline creates both files automatically. Background publication preserves manually edited and removed tracks.
+
+## Who may use it
+
+`MaveCore.UsageLimits.can_use_audio_description?/1` is an optional host callback.
+Without a host policy every space may turn the feature on. A host can refuse with
+`{:error, reason}` and provide `restriction_copy(:audio_description, reason)`;
+the dashboard then shows the notice and keeps the toggle off. Turning the feature
+off is always allowed. The check is repeated for every new run, so a space that
+loses access stops getting audio descriptions without changing its setting.
+
+## Providers and configuration
+
+The default providers are Scaleway Generative APIs and Mistral. Video stills,
+speech timings and description text are sent to the configured providers.
+
+| Variable | Default | Purpose |
+| --- | --- | --- |
+| `SCALEWAY_API_KEY` | — | Scaleway IAM application with only `GenerativeApisModelAccess` |
+| `MISTRAL_API_KEY` | — | Already used for transcription; also used for text-to-speech |
+| `MAVE_AUDIO_DESCRIPTION_ENDPOINT` | `https://api.scaleway.ai/v1/chat/completions` | Override with `https://api.scaleway.ai/<project-id>/v1/chat/completions` for a project-scoped key |
+| `MAVE_AUDIO_DESCRIPTION_MODEL` | `gemma-4-26b-a4b-it` | Describe and translate model |
+| `MAVE_AUDIO_DESCRIPTION_REASONING_EFFORT` | `none` | Scaleway `reasoning_effort`; with reasoning, Gemma runs out of tokens |
+| `MAVE_AUDIO_DESCRIPTION_TTS_MODEL` | `voxtral-mini-tts-3-0` | Mistral text-to-speech model |
+| `MAVE_AUDIO_DESCRIPTION_VOICES` | `en:gb_jane_neutral,nl:voice_3145` | Voice per language |
+| `MAVE_AUDIO_DESCRIPTION_VOICE` | `gb_jane_neutral` | Voice for languages without one |
+| `MAVE_AUDIO_DESCRIPTION_MAX_SECONDS` | `600` | Longest video that gets descriptions |
+
+The default voices are Jane (English) and Lotte (Dutch), both presets. Lotte
+requires Voxtral 3; the older `voxtral-mini-tts-2603` model rejects this voice.
+When overriding the model or voices, verify that they are compatible. See
+[Mistral's voice catalog](https://docs.mistral.ai/studio/audio/text-to-speech/voices).
+
+Scope the Scaleway application's `GenerativeApisModelAccess` policy to the
+intended project and use that project's endpoint above. Setting a key's default
+project alone does not restrict its permissions. Prefer expiring credentials
+and keep them in the runtime environment, outside version control.
+
+Tests replace the providers through application config:
+
+```elixir
+config :mave_core, :audio_description,
+  chat_provider: MyChatStub,     # chat_json(system, content, opts)
+  speech_provider: MySpeechStub, # speak(text, language) -> {:ok, wav_bytes}
+  max_seconds: 600
+```

docs/flow_engine.md

@@ -117,6 +117,14 @@ New runs select the current built-in preset version. Existing runs and pinned
 custom versions do not gain steps automatically; reprocess with an updated flow
 when peaks are needed for older media.
 
+## Audio description
+
+The publishing presets include optional background steps that generate an audio
+description (a full-mix MP3 and a descriptions VTT, in English and Dutch) for
+spaces that opted in. They skip themselves unless the run input has
+`audio_description`, and nothing in the publish path waits for them. See
+[audio description](audio_description.md).
+
 ## Execution and recovery
 
 By default:

lib/mave_core/analytics/sessions.ex

@@ -565,7 +565,7 @@ defmodule MaveCore.Analytics.Sessions do
     :ok
   end
 
-  @doc "Hide retired analytics immediately; obsolete generations are collected separately."
+  @doc "Hide prepared analytics immediately; physical cleanup runs after the reader grace period."
   def invalidate(space_hash) do
     with :ok <- validate_hash(space_hash), do: invalidate_transaction(space_hash)
   end
@@ -579,43 +579,22 @@ defmodule MaveCore.Analytics.Sessions do
 
   defp invalidate_locked(space) do
     lock!(space)
-    Repo.query!("DELETE FROM analytics_session_buckets WHERE space_hash = $1", [space])
 
-    with {:ok, _} <-
-           ch(
-             "ALTER TABLE analytics_sessions DELETE WHERE space_hash = {$0:String} SETTINGS mutations_sync = 1",
-             [space],
-             60_000
-           ),
-         {:ok, _} <-
-           ch(
-             "ALTER TABLE analytics_session_years DELETE WHERE space_hash = {$0:String} SETTINGS mutations_sync = 1",
-             [space],
-             60_000
-           ),
-         {:ok, _} <-
-           ch(
-             "ALTER TABLE analytics_session_reports DELETE WHERE space_hash = {$0:String} SETTINGS mutations_sync = 1",
-             [space],
-             60_000
-           ),
-         {:ok, _} <-
-           ch(
-             "ALTER TABLE analytics_overview_days DELETE WHERE space_hash = {$0:String} SETTINGS mutations_sync = 1",
-             [space],
-             60_000
-           ),
-         {:ok, _} <-
-           ch(
-             "ALTER TABLE analytics_overview_views DELETE WHERE space_hash = {$0:String} SETTINGS mutations_sync = 1",
-             [space],
-             60_000
-           ) do
-      Repo.query!("DELETE FROM analytics_session_retirements WHERE space_hash = $1", [space])
-      :ok
-    else
-      _ -> {:error, :analytics_unavailable}
-    end
+    # Unpublish and retain a durable cleanup record in one transaction. Readers
+    # already using a generation keep it for the grace period; new readers cannot
+    # select it. A ClickHouse outage must not block this visibility change.
+    Repo.query!(
+      """
+      INSERT INTO analytics_session_retirements (space_hash, generation, retired_at)
+      SELECT space_hash, generation, now() FROM analytics_session_buckets
+      WHERE space_hash = $1
+      ON CONFLICT (space_hash, generation) DO NOTHING
+      """,
+      [space]
+    )
+
+    Repo.query!("DELETE FROM analytics_session_buckets WHERE space_hash = $1", [space])
+    :ok
   end
 
   @doc "Delete retired generations after the reader grace period, in batches per space."