diff --git a/api-reference/asyncapi-asr-realtime.yml b/api-reference/asyncapi-asr-realtime.yml new file mode 100644 index 00000000..a35c61ee --- /dev/null +++ b/api-reference/asyncapi-asr-realtime.yml @@ -0,0 +1,1283 @@ +asyncapi: 3.0.0 + +info: + title: Fish Audio Realtime Speech-to-Text WebSocket API + version: 1.0.0 + description: | + Realtime speech-to-text streaming via WebSocket using MessagePack serialization. + + ## Protocol Overview + + The realtime speech-to-text endpoint streams audio in and transcripts out: + + 1. **Client** connects to `/v1/asr/realtime` with auth headers + 2. **Client** sends `config`, and the **Server** answers with `session_started` + 3. **Client** streams `audio` messages + 4. **Server** sends turn events (`speech_start`, `user_turn_taken`, `eager_end_of_turn`) and a `committed` transcript per turn + 5. **Client** sends `audio_done`; the **Server** sends `session_done` and closes the connection + + ## Message Serialization + + All messages are serialized using **MessagePack** (binary format) before sending over the WebSocket. + + ## Common Use Cases + + - Voice agents that listen and respond in real time + - Live captions + - Voice commands + - Phone calls + +servers: + production: + host: api.fish.audio + protocol: wss + description: Fish Audio production WebSocket server + security: + - $ref: "#/components/securitySchemes/bearerAuth" + +channels: + asrRealtime: + address: /v1/asr/realtime + messages: + config: + $ref: "#/components/messages/AsrConfig" + audio: + $ref: "#/components/messages/AsrAudio" + finalize: + $ref: "#/components/messages/AsrFinalize" + configure: + $ref: "#/components/messages/AsrConfigure" + contextAdd: + $ref: "#/components/messages/AsrContextAdd" + keepalive: + $ref: "#/components/messages/AsrKeepalive" + audioDone: + $ref: "#/components/messages/AsrAudioDone" + agentSpeaking: + $ref: "#/components/messages/AsrAgentSpeaking" + sessionStarted: + $ref: "#/components/messages/AsrSessionStarted" + speechStart: + $ref: "#/components/messages/AsrSpeechStart" + userTurnTaken: + $ref: "#/components/messages/AsrUserTurnTaken" + draft: + $ref: "#/components/messages/AsrDraft" + eagerEndOfTurn: + $ref: "#/components/messages/AsrEagerEndOfTurn" + turnResumed: + $ref: "#/components/messages/AsrTurnResumed" + userBackchannel: + $ref: "#/components/messages/AsrUserBackchannel" + committed: + $ref: "#/components/messages/AsrCommitted" + vad: + $ref: "#/components/messages/AsrVad" + eotProb: + $ref: "#/components/messages/AsrEotProb" + configured: + $ref: "#/components/messages/AsrConfigured" + progress: + $ref: "#/components/messages/AsrProgress" + error: + $ref: "#/components/messages/AsrError" + sessionDone: + $ref: "#/components/messages/AsrSessionDone" + bindings: + ws: + headers: + type: object + properties: + model: + type: string + enum: + - transcribe-1-pro-realtime + default: transcribe-1-pro-realtime + description: Realtime speech-to-text model. Optional; `transcribe-1-pro-realtime` is the default and the only realtime model. Any other value is refused with HTTP 400. + description: | + Realtime speech-to-text channel. Clients stream audio and receive + transcripts turn by turn, with turn detection, optional live drafts and + conversation context. + + ## Connection Headers + - `Authorization: Bearer ` - Required for authentication (see security section) + - `model: transcribe-1-pro-realtime` - Optional; the default + + Every message in both directions is one MessagePack map in a binary + frame, with a `type` field. Send `config` first, wait for + `session_started`, then stream `audio`. Ignore fields and message types + you do not know: version 1 only adds them. + +operations: + receiveAsrClientMessages: + action: receive + channel: + $ref: "#/channels/asrRealtime" + messages: + - $ref: "#/channels/asrRealtime/messages/config" + - $ref: "#/channels/asrRealtime/messages/audio" + - $ref: "#/channels/asrRealtime/messages/finalize" + - $ref: "#/channels/asrRealtime/messages/configure" + - $ref: "#/channels/asrRealtime/messages/contextAdd" + - $ref: "#/channels/asrRealtime/messages/keepalive" + - $ref: "#/channels/asrRealtime/messages/audioDone" + - $ref: "#/channels/asrRealtime/messages/agentSpeaking" + description: | + Server receives configuration, audio and control messages from the client. + + **Message Sequence:** + 1. Client sends `config` once, as the first message, within 10 seconds of connecting + 2. Client streams `audio` after `session_started`, 80–100 ms per message + 3. Client can send `finalize`, `configure`, `context_add`, `keepalive` and `agent_speaking` at any time + 4. Client sends `audio_done` when there is no more audio + + sendAsrServerMessages: + action: send + channel: + $ref: "#/channels/asrRealtime" + messages: + - $ref: "#/channels/asrRealtime/messages/sessionStarted" + - $ref: "#/channels/asrRealtime/messages/speechStart" + - $ref: "#/channels/asrRealtime/messages/userTurnTaken" + - $ref: "#/channels/asrRealtime/messages/draft" + - $ref: "#/channels/asrRealtime/messages/eagerEndOfTurn" + - $ref: "#/channels/asrRealtime/messages/turnResumed" + - $ref: "#/channels/asrRealtime/messages/userBackchannel" + - $ref: "#/channels/asrRealtime/messages/committed" + - $ref: "#/channels/asrRealtime/messages/vad" + - $ref: "#/channels/asrRealtime/messages/eotProb" + - $ref: "#/channels/asrRealtime/messages/configured" + - $ref: "#/channels/asrRealtime/messages/progress" + - $ref: "#/channels/asrRealtime/messages/error" + - $ref: "#/channels/asrRealtime/messages/sessionDone" + description: | + Server sends session, turn and transcript messages to the client. + + **Event Flow:** + - Server sends `session_started` once, after a valid `config` + - For each turn: `speech_start`, then optional `draft` messages, `user_turn_taken` (auto mode), optional `eager_end_of_turn` / `turn_resumed`, and exactly one `committed` + - Server sends `user_backchannel` in auto mode while the client reports `agent_speaking: true` + - Server sends `session_done` after `audio_done`, then closes with 1000 + - Clients should ignore unknown messages and fields to support future protocol extensions + +components: + messages: + AsrConfig: + name: config + title: Configure the Session + contentType: application/msgpack + payload: + type: object + required: + - type + - audio + properties: + type: + type: string + const: config + description: Message type identifier + protocol_version: + type: integer + const: 1 + default: 1 + description: Optional. Only `1` is valid. + audio: + $ref: "#/components/schemas/AsrAudioFormat" + turn_detection: + $ref: "#/components/schemas/AsrTurnDetection" + results: + $ref: "#/components/schemas/AsrResults" + context: + $ref: "#/components/schemas/AsrContext" + client_reference_id: + type: string + maxLength: 256 + description: Your own ID for the session. Recorded with the session; not sent back. + description: | + The first message on the connection. Send it within 10 seconds of + connecting, then wait for `session_started` before sending audio. + An invalid `config` is answered with an `error` (`invalid_config` or + `unsupported_audio`) and close code 4400. + examples: + - name: Voice agent + payload: + type: config + audio: + format: pcm_s16le + sample_rate: 16000 + channels: 1 + turn_detection: + mode: auto + eager: true + context: + terms: + - Fish Audio + - name: Live captions + payload: + type: config + audio: + format: pcm_s16le + sample_rate: 48000 + channels: 1 + results: + drafts: true + diarize: false + + AsrAudio: + name: audio + title: Send Audio + contentType: application/msgpack + payload: + type: object + required: + - type + - data + properties: + type: + type: string + const: audio + description: Message type identifier + data: + type: string + format: binary + description: PCM bytes in the configured format, as MessagePack `bin`. Any length; an odd trailing byte waits for the next message. + description: | + Audio in the format from `config`. Send it after `session_started`; + 80–100 ms per message is a good size. A message can be at most 1 MiB. + examples: + - payload: + type: audio + data: + + AsrFinalize: + name: finalize + title: Commit the Current Turn + contentType: application/msgpack + payload: + type: object + required: + - type + properties: + type: + type: string + const: finalize + description: Message type identifier + description: | + Commits the current turn now, with `reason: "finalize"`. Ignored when + no turn is open. In auto mode the turn it ends is always a user turn. + Use it for push-to-talk (manual mode) when the button is released. + examples: + - payload: + type: finalize + + AsrConfigure: + name: configure + title: Change Settings + contentType: application/msgpack + payload: + type: object + required: + - type + properties: + type: + type: string + const: configure + description: Message type identifier + turn_detection: + $ref: "#/components/schemas/AsrTurnDetectionUpdate" + results: + $ref: "#/components/schemas/AsrResultsUpdate" + context: + type: object + properties: + terms: + type: array + maxItems: 100 + items: + type: string + maxLength: 32 + description: Replaces the list of terms. + description: | + Only `terms` is accepted here. `history` is ignored in + `configure`; send earlier lines with `context_add`. + description: | + Changes settings during the session. Answered by `configured`. Every + field is optional, and a field you leave out keeps its current value. + The one exception: switching `mode` without `silence_timeout_ms` + resets it to the new mode's default. If any value is invalid, + nothing in the message is applied. When `mode` changes to or from + `auto`, `configured` can take a few seconds. + examples: + - payload: + type: configure + turn_detection: + mode: vad + silence_timeout_ms: 900 + - payload: + type: configure + context: + terms: + - Fish Audio + - Main Street + + AsrContextAdd: + name: context_add + title: Add Conversation Context + contentType: application/msgpack + payload: + type: object + required: + - type + - items + properties: + type: + type: string + const: context_add + description: Message type identifier + items: + type: array + maxItems: 32 + items: + $ref: "#/components/schemas/AsrContextItem" + description: Conversation items, oldest first. Send each line your agent speaks as an `assistant` item. + reset_history: + type: boolean + default: false + description: First drop the items you added before. + description: | + Adds conversation context. Committed user turns are added + automatically; send what the other party (your agent) said, ideally + right after the `committed` it answers. No reply. + examples: + - payload: + type: context_add + items: + - role: assistant + text: Sure. Which day works for you? + + AsrKeepalive: + name: keepalive + title: Keep the Session Open + contentType: application/msgpack + payload: + type: object + required: + - type + properties: + type: + type: string + const: keepalive + description: Message type identifier + description: | + Keeps an idle session open. A session closes after 30 seconds without + any client message. + examples: + - payload: + type: keepalive + + AsrAudioDone: + name: audio_done + title: End of Audio + contentType: application/msgpack + payload: + type: object + required: + - type + properties: + type: + type: string + const: audio_done + description: Message type identifier + description: | + No more audio. The server commits the open turn + (`reason: "audio_done"`), sends `session_done`, and closes with 1000. + examples: + - payload: + type: audio_done + + AsrAgentSpeaking: + name: agent_speaking + title: Agent Speaking State + contentType: application/msgpack + payload: + type: object + required: + - type + - speaking + properties: + type: + type: string + const: agent_speaking + description: Message type identifier + speaking: + type: boolean + description: | + `true` when you commit to playing a reply (typically on + `committed`); `false` when its last sample has played or you + stop it. Default: not speaking. + description: | + For voice agents in auto mode. While the agent is speaking, a short + acknowledgement from the user produces `user_backchannel` instead of + `user_turn_taken`. Applies at its position in the audio stream; send + it on the same connection as your audio, only when the value changes. + No effect in vad and manual mode. + examples: + - payload: + type: agent_speaking + speaking: true + + AsrSessionStarted: + name: session_started + title: Session Started + contentType: application/msgpack + payload: + type: object + required: + - type + - seq + - session_id + - request_id + - config + - limits + properties: + type: + type: string + const: session_started + description: Message type identifier + seq: + type: integer + description: Message sequence number, increasing by one per server message. + session_id: + type: string + description: The session's ID. + request_id: + type: string + description: Same as the `x-request-id` response header. Include it when you contact support. + config: + type: object + description: The effective configuration, with defaults filled in. + limits: + $ref: "#/components/schemas/AsrLimits" + description: | + The session is ready. Send audio after this message. + examples: + - payload: + type: session_started + seq: 0 + session_id: s_ab12cd34 + request_id: 0b6f4c1e-7d2a-4e8b-9f3c-5a1d2e3f4b6c + config: + protocol_version: 1 + model: transcribe-1-pro-realtime + audio: + format: pcm_s16le + sample_rate: 16000 + channels: 1 + turn_detection: + mode: auto + eager: true + silence_timeout_ms: 5000 + results: + drafts: true + draft_interval_ms: 250 + ignore_timestamps: true + diarize: true + vad_events: false + eot_events: false + limits: + max_session_ms: 3600000 + max_turn_ms: 60000 + max_frame_bytes: 1048576 + + AsrSpeechStart: + name: speech_start + title: Turn Opened + contentType: application/msgpack + payload: + type: object + required: + - type + - seq + - turn_index + - audio_ms + properties: + type: + type: string + const: speech_start + description: Message type identifier + seq: + type: integer + turn_index: + type: integer + description: The turn that opened, counting from 0. + audio_ms: + type: integer + description: Where the turn starts, slightly before the speech. + description: | + A turn opened on voice activity. In auto mode, noise and other voices + open turns too: wait for `user_turn_taken` before treating it as the + user's turn. + examples: + - payload: + type: speech_start + seq: 1 + turn_index: 0 + audio_ms: 300 + + AsrUserTurnTaken: + name: user_turn_taken + title: User Took the Turn + contentType: application/msgpack + payload: + type: object + required: + - type + - seq + - turn_index + - audio_ms + properties: + type: + type: string + const: user_turn_taken + description: Message type identifier + seq: + type: integer + turn_index: + type: integer + audio_ms: + type: integer + description: Where the server judged that the user took the turn. + description: | + Auto mode only: the server judged that the user is taking their turn. + At most once per turn, before any `eager_end_of_turn`. For a voice + agent that is speaking, this is the signal to stop (barge-in). + examples: + - payload: + type: user_turn_taken + seq: 3 + turn_index: 0 + audio_ms: 1280 + + AsrDraft: + name: draft + title: Draft Transcript + contentType: application/msgpack + payload: + type: object + required: + - type + - seq + - turn_index + - text + - audio_ms + properties: + type: + type: string + const: draft + description: Message type identifier + seq: + type: integer + turn_index: + type: integer + text: + type: string + description: The whole turn so far. Replaces the previous draft. Not final. + language: + type: string + description: Optional. English name of the detected language. + language_code: + type: string + description: Optional. ISO 639-1 code of the detected language. + audio_ms: + type: integer + description: The audio position the draft covers up to. + audio_received_ms: + type: integer + audio_processed_ms: + type: integer + audio_committed_ms: + type: integer + description: | + Live draft of the open turn, with `results.drafts: true`. Best effort, + at most every `draft_interval_ms`. + examples: + - payload: + type: draft + seq: 4 + turn_index: 0 + text: what's the weather like + language: English + language_code: en + audio_ms: 1800 + audio_received_ms: 1900 + audio_processed_ms: 1800 + audio_committed_ms: 0 + + AsrEagerEndOfTurn: + name: eager_end_of_turn + title: Eager End of Turn + contentType: application/msgpack + payload: + type: object + required: + - type + - seq + - turn_index + - text + properties: + type: + type: string + const: eager_end_of_turn + description: Message type identifier + seq: + type: integer + turn_index: + type: integer + text: + type: string + description: Exactly what `committed` will say unless `turn_resumed` follows. Can be empty. + language: + type: string + language_code: + type: string + audio_ms: + type: integer + description: Optional. The audio position of the signal. + description: | + With `turn_detection.eager: true` in auto mode, after + `user_turn_taken`: the turn has probably ended. Start preparing your + response; `turn_resumed` cancels it. Not every turn has one. + examples: + - payload: + type: eager_end_of_turn + seq: 5 + turn_index: 0 + text: What's the weather like? + language: English + language_code: en + audio_ms: 2240 + + AsrTurnResumed: + name: turn_resumed + title: Turn Resumed + contentType: application/msgpack + payload: + type: object + required: + - type + - seq + - turn_index + properties: + type: + type: string + const: turn_resumed + description: Message type identifier + seq: + type: integer + turn_index: + type: integer + description: | + The speaker went on after `eager_end_of_turn`: discard that text. If + you already started playing a reply, stop it and send + `agent_speaking: false`. + examples: + - payload: + type: turn_resumed + seq: 6 + turn_index: 0 + + AsrUserBackchannel: + name: user_backchannel + title: User Backchannel + contentType: application/msgpack + payload: + type: object + required: + - type + - seq + - turn_index + - audio_ms + properties: + type: + type: string + const: user_backchannel + description: Message type identifier + seq: + type: integer + turn_index: + type: integer + description: The open turn, or the index the next turn will get when no turn is open. + audio_ms: + type: integer + description: Where the acknowledgement is in the audio. + description: | + Auto mode only, while your agent is speaking: the user acknowledged + ("mm-hm", "yeah") without taking the turn. Keep talking. + examples: + - payload: + type: user_backchannel + seq: 8 + turn_index: 1 + audio_ms: 3840 + + AsrCommitted: + name: committed + title: Committed Turn + contentType: application/msgpack + payload: + type: object + required: + - type + - seq + - turn_index + - text + - audio_start_ms + - audio_end_ms + - reason + - degraded + - user_turn + - turn_kind + properties: + type: + type: string + const: committed + description: Message type identifier + seq: + type: integer + turn_index: + type: integer + text: + type: string + description: The final transcript of the turn. Can contain speaker tags such as `<|speaker:0|>`. Can be empty. + words: + type: array + items: + $ref: "#/components/schemas/AsrWord" + description: Present when `results.ignore_timestamps` is `false` and the commit is not degraded. + language: + type: string + language_code: + type: string + audio_start_ms: + type: integer + audio_end_ms: + type: integer + reason: + type: string + enum: + - eot + - finalize + - silence_timeout + - max_turn_duration + - audio_done + description: Why the turn ended. Treat an unknown value like `eot`. + degraded: + type: boolean + description: "`true` when the final pass failed and the text comes from the last draft." + user_turn: + type: boolean + description: "`true` for the user's turn. Decide whether to answer by this field. Treat a missing value as `true`." + turn_kind: + type: string + enum: + - user + - backchannel + - other + description: "`user` exactly when `user_turn` is true. Treat an unknown value like `other` when `user_turn` is false." + audio_received_ms: + type: integer + audio_processed_ms: + type: integer + audio_committed_ms: + type: integer + description: | + The final transcript of a turn. Every turn has exactly one, and nothing + about the turn follows it. + examples: + - payload: + type: committed + seq: 6 + turn_index: 0 + text: What's the weather like? + language: English + language_code: en + audio_start_ms: 300 + audio_end_ms: 2880 + reason: eot + degraded: false + user_turn: true + turn_kind: user + audio_received_ms: 3000 + audio_processed_ms: 2900 + audio_committed_ms: 2880 + + AsrVad: + name: vad + title: Voice Activity + contentType: application/msgpack + payload: + type: object + required: + - type + - seq + - audio_ms + - score + properties: + type: + type: string + const: vad + description: Message type identifier + seq: + type: integer + audio_ms: + type: integer + score: + type: number + minimum: 0 + maximum: 1 + description: Voice activity. + description: | + Voice activity, up to 10 per second, with `results.vad_events: true`. + examples: + - payload: + type: vad + seq: 12 + audio_ms: 1100 + score: 0.93 + + AsrEotProb: + name: eot_prob + title: End-of-Turn Likelihood + contentType: application/msgpack + payload: + type: object + required: + - type + - seq + - turn_index + - audio_ms + - prob + properties: + type: + type: string + const: eot_prob + description: Message type identifier + seq: + type: integer + turn_index: + type: integer + audio_ms: + type: integer + prob: + type: number + minimum: 0 + maximum: 1 + description: | + End-of-turn likelihood for display, with `results.eot_events: true`. + examples: + - payload: + type: eot_prob + seq: 13 + turn_index: 0 + audio_ms: 2200 + prob: 0.42 + + AsrConfigured: + name: configured + title: Configure Result + contentType: application/msgpack + payload: + type: object + required: + - type + - seq + - ok + - config + properties: + type: + type: string + const: configured + description: Message type identifier + seq: + type: integer + ok: + type: boolean + description: "`false` means nothing changed." + config: + type: object + description: The configuration in effect. + message: + type: string + description: Present when `ok` is `false`. Why, in English, for humans. + description: | + The reply to `configure`. + examples: + - payload: + type: configured + seq: 1 + ok: false + config: + turn_detection: + mode: auto + eager: false + silence_timeout_ms: 5000 + message: turn_detection.silence_timeout_ms must be between 3000 and 60000 in auto mode + + AsrProgress: + name: progress + title: Progress + contentType: application/msgpack + payload: + type: object + required: + - type + - seq + properties: + type: + type: string + const: progress + description: Message type identifier + seq: + type: integer + audio_received_ms: + type: integer + audio_processed_ms: + type: integer + audio_committed_ms: + type: integer + description: | + Watermarks only, every 5 seconds while the client sends nothing. + examples: + - payload: + type: progress + seq: 30 + audio_received_ms: 12000 + audio_processed_ms: 12000 + audio_committed_ms: 9600 + + AsrError: + name: error + title: Error + contentType: application/msgpack + payload: + type: object + required: + - type + - seq + - code + - error_type + - message + - request_id + - recoverable + properties: + type: + type: string + const: error + description: Message type identifier + seq: + type: integer + code: + type: integer + enum: + - 400 + - 408 + - 413 + - 500 + - 503 + - 504 + description: HTTP-style status. + error_type: + type: string + enum: + - invalid_config + - unsupported_audio + - invalid_request + - idle_timeout + - max_duration_exceeded + - frame_too_large + - engine_error + - internal + - server_restarting + description: Match on this field, not on `message`. + message: + type: string + description: A short English description, for humans. Do not parse it. + request_id: + type: string + recoverable: + type: boolean + description: | + `true` when the session continues, except for + `server_restarting`, which commits the open turn and closes + with 1012 so you can reconnect. + description: | + An error. When `recoverable` is `false`, the server closes the + connection (close codes 4400, 4408, 4413, 4500 or 1009). + `server_restarting` is recoverable but still closes with 1012: + reconnect for a new session. + examples: + - payload: + type: error + seq: 1 + code: 400 + error_type: unsupported_audio + message: audio.sample_rate must be one of 8000, 12000, 16000, 22050, 24000, 32000, 44100, 48000 + request_id: 0b6f4c1e-7d2a-4e8b-9f3c-5a1d2e3f4b6c + recoverable: false + + AsrSessionDone: + name: session_done + title: Session Done + contentType: application/msgpack + payload: + type: object + required: + - type + - seq + - stats + properties: + type: + type: string + const: session_done + description: Message type identifier + seq: + type: integer + stats: + type: object + properties: + audio_ms: + type: integer + description: Audio received in the session. + turns: + type: integer + description: Every committed turn, user turn or not. + request_id: + type: string + description: | + Sent after `audio_done`, just before the server closes with 1000. + examples: + - payload: + type: session_done + seq: 12 + stats: + audio_ms: 8960 + turns: 3 + request_id: 0b6f4c1e-7d2a-4e8b-9f3c-5a1d2e3f4b6c + + schemas: + AsrAudioFormat: + type: object + required: + - format + - sample_rate + - channels + properties: + format: + type: string + const: pcm_s16le + description: 16-bit little-endian PCM. + sample_rate: + type: integer + enum: + - 8000 + - 12000 + - 16000 + - 22050 + - 24000 + - 32000 + - 44100 + - 48000 + description: Sample rate in Hz. Rates other than 16000 are converted on the server; you never need to resample. + channels: + type: integer + const: 1 + description: Mono only. + + AsrTurnDetection: + type: object + properties: + mode: + type: string + enum: + - auto + - vad + - manual + default: auto + description: | + auto: the server's turn detection decides when the user has + finished, and tells user turns from noise and other voices. + vad: a turn ends after `silence_timeout_ms` of silence. + manual: a turn ends when you send `finalize`. + eager: + type: boolean + default: false + description: Send `eager_end_of_turn` in auto mode. + silence_timeout_ms: + type: integer + description: | + auto: default 5000, allowed 3000–60000 (a safety net for user + turns). vad: default 700, allowed 300–60000 (the silence that + ends a turn). manual: default 60000, allowed 300–60000 (a safety + net). Checked against the mode it will run under. + + AsrResults: + type: object + properties: + drafts: + type: boolean + default: false + description: Send live `draft` messages. + draft_interval_ms: + type: integer + minimum: 0 + default: 250 + description: Minimum time between drafts. Smaller values count as 250. + ignore_timestamps: + type: boolean + default: true + description: "`false` adds `words` with timings to `committed`." + diarize: + oneOf: + - type: boolean + - type: string + enum: + - auto + - "true" + - "false" + default: true + description: Speaker tags in the text. `auto` is the same as `true`; `false` removes them. The strings `"true"` and `"false"` are accepted too. + vad_events: + type: boolean + default: false + description: Send `vad` messages. + eot_events: + type: boolean + default: false + description: Send `eot_prob` messages. + + AsrTurnDetectionUpdate: + description: Turn detection changes in `configure`. Omitted fields keep their current values. + type: object + properties: + mode: + type: string + enum: + - auto + - vad + - manual + description: | + auto: the server's turn detection decides when the user has + finished, and tells user turns from noise and other voices. + vad: a turn ends after `silence_timeout_ms` of silence. + manual: a turn ends when you send `finalize`. + eager: + type: boolean + description: Send `eager_end_of_turn` in auto mode. + silence_timeout_ms: + type: integer + description: | + auto: default 5000, allowed 3000–60000 (a safety net for user + turns). vad: default 700, allowed 300–60000 (the silence that + ends a turn). manual: default 60000, allowed 300–60000 (a safety + net). Checked against the mode it will run under. + + AsrResultsUpdate: + description: Result changes in `configure`. Omitted fields keep their current values. + type: object + properties: + drafts: + type: boolean + description: Send live `draft` messages. + draft_interval_ms: + type: integer + minimum: 0 + description: Minimum time between drafts. Smaller values count as 250. + ignore_timestamps: + type: boolean + description: "`false` adds `words` with timings to `committed`." + diarize: + oneOf: + - type: boolean + - type: string + enum: + - auto + - "true" + - "false" + description: Speaker tags in the text. `auto` is the same as `true`; `false` removes them. The strings `"true"` and `"false"` are accepted too. + vad_events: + type: boolean + description: Send `vad` messages. + eot_events: + type: boolean + description: Send `eot_prob` messages. + + AsrContext: + type: object + properties: + history: + type: array + maxItems: 32 + items: + $ref: "#/components/schemas/AsrContextItem" + description: Earlier conversation, oldest first. + terms: + type: array + maxItems: 100 + items: + type: string + maxLength: 32 + description: Names, products and jargon to recognize. + + AsrContextItem: + type: object + required: + - role + - text + properties: + role: + type: string + enum: + - user + - assistant + text: + type: string + maxLength: 2000 + + AsrWord: + type: object + required: + - text + - start_ms + - end_ms + properties: + text: + type: string + start_ms: + type: integer + end_ms: + type: integer + + AsrLimits: + type: object + properties: + max_session_ms: + type: integer + description: Maximum session length (3600000). + max_turn_ms: + type: integer + description: Maximum turn length (60000). + max_frame_bytes: + type: integer + description: Maximum message size (1048576). + + securitySchemes: + bearerAuth: + type: http + scheme: bearer + description: | + API key authentication using Bearer token. + + Get your API key from https://fish.audio/app/api-keys + + Pass the token in the Authorization header: + `Authorization: Bearer YOUR_API_KEY` diff --git a/api-reference/endpoint/websocket/asr-realtime.mdx b/api-reference/endpoint/websocket/asr-realtime.mdx new file mode 100644 index 00000000..ae0a21a5 --- /dev/null +++ b/api-reference/endpoint/websocket/asr-realtime.mdx @@ -0,0 +1,521 @@ +--- +asyncapi: "/api-reference/asyncapi-asr-realtime.yml asrRealtime" +title: "WebSocket Realtime Speech to Text" +description: "Stream audio over a WebSocket and receive transcripts turn by turn, with turn detection, live drafts and conversation context" +icon: "webhook" +iconType: "solid" +hidden: true +noindex: true +--- + + + Connect to `wss://api.fish.audio/v1/asr/realtime` with a WebSocket client. + Every message in both directions is one MessagePack map in a binary frame. A + normal HTTP GET request does not start a session. For an introduction, see + [Realtime Speech to Text](/features/speech-to-text/realtime). + + + + Realtime speech to text is in preview and is enabled per team. Contact + [support@fish.audio](mailto:support@fish.audio) to turn it on for your team. + Until then, connecting returns `403`. + + +## Before you start + +You need a [Fish Audio API key](/developer-guide/getting-started/api-key) and a WebSocket client that can set request headers. The API key travels in a request header, so connect from your server, not from a browser; to capture audio in a browser, relay it through your server as in [Transcribe a live audio stream](/developer-guide/sdk-guide/cookbook/realtime-transcription#capture-from-a-browser-microphone). The examples on these pages use Python with `websockets` and `msgpack`, and Node.js with `ws` and `@msgpack/msgpack`. + +This endpoint is separate from the realtime endpoints of the [compatible APIs](/developer-guide/compat/capabilities), which use manual commit and have no server-side turn detection. + +## Connect + +```text +GET wss://api.fish.audio/v1/asr/realtime +Authorization: Bearer +model: transcribe-1-pro-realtime +``` + +| Header | Required | Description | +| --------------- | -------- | -------------------------------------------------------------------------------------------------- | +| `Authorization` | Yes | `Bearer `. | +| `model` | No | `transcribe-1-pro-realtime`, the default and the only realtime model. Any other value returns 400. | + +Each open connection holds one of your team's concurrent session slots until it closes. A session cannot be resumed: a new connection is a new session. + +If the server refuses the upgrade, it answers with a plain HTTP response whose JSON body is `{"status": , "message": }`: + +| Status | Meaning | What to do | +| ------ | --------------------------------------------------------------------- | --------------------------------------------------------- | +| 400 | The `model` header names another model. | Send `transcribe-1-pro-realtime` or leave the header out. | +| 401 | The API key is missing or invalid. | Check the key. | +| 402 | Insufficient API credit. | Top up your balance. | +| 403 | Realtime speech to text is not enabled for your team. | Contact support (see the note above). | +| 429 | Your team is at its concurrency limit, or the service is at capacity. | Retry with backoff. Honor `Retry-After` when present. | +| 503 | The service is restarting. | Retry. | + +The `101 Switching Protocols` response carries an `x-request-id` header, and so do the 400 (model), capacity 429 and 503 refusals. It is the same value as `request_id` in `session_started`, `error` and `session_done`. Log it, and include it when you contact support. Refusals for the key, balance, access and your concurrency limit (401, 402, 403 and concurrency 429) happen before a session is assigned and carry no `x-request-id`. + +## Framing + +- Every message is **one binary WebSocket frame holding one MessagePack map** with a `type` field. Audio bytes are MessagePack `bin`. Do not send text frames. +- Every server message carries `seq`, which increases by one per message within the session. +- Times are `*_ms` integers: milliseconds of session audio since the first audio byte, in your audio's own time at the `sample_rate` you configured. +- WebSocket ping and pong frames are allowed at any time, including before `config`. The server answers pings, and a ping counts as client activity for the 30-second idle limit. Pings do not extend the 10-second deadline for the first `config`. + +## Session flow + +```text +Client → config first message, within 10 s of connecting +Server → session_started +Client → audio, audio, ... plus finalize, configure, context_add, keepalive, agent_speaking at any time + +Per turn: +Server → speech_start + draft* only with results.drafts: true + user_turn_taken auto mode only + eager_end_of_turn with turn_detection.eager: true; turn_resumed cancels it + committed the final transcript of the turn +Server → user_backchannel auto mode, while your agent is speaking + +Client → audio_done +Server → committed the open turn, reason "audio_done" + session_done + close 1000 +``` + +In `auto` mode, `draft` messages can come before and after `user_turn_taken`, and `eager_end_of_turn` followed by `turn_resumed` can repeat before the turn commits. After `audio_done`, the server first finishes its turn decisions on the audio already sent, so you can still receive `speech_start`, `user_turn_taken` and `committed` for earlier turns. + +## Client messages + +### `config` + +The first message on the connection. Send it within 10 seconds of connecting, then wait for `session_started` before you send audio. + +```json +{ + "type": "config", + "audio": { "format": "pcm_s16le", "sample_rate": 16000, "channels": 1 }, + "turn_detection": { "mode": "auto", "eager": true }, + "results": { "drafts": true, "ignore_timestamps": false }, + "context": { + "history": [{ "role": "assistant", "text": "Hi, how can I help?" }], + "terms": ["Fish Audio", "transcribe-1-pro-realtime"] + }, + "client_reference_id": "call-8f2c" +} +``` + +| Field | Required | Default | Description | +| ----------------------------------- | -------- | --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `protocol_version` | No | `1` | Only `1` is valid. | +| `audio.format` | Yes | | `pcm_s16le`: 16-bit little-endian PCM. | +| `audio.sample_rate` | Yes | | One of 8000, 12000, 16000, 22050, 24000, 32000, 44100, 48000. See [Audio format](#audio-format). | +| `audio.channels` | Yes | | `1` (mono). | +| `turn_detection.mode` | No | `"auto"` | `"auto"`, `"vad"` or `"manual"`. See [Turn detection modes](#turn-detection-modes). | +| `turn_detection.eager` | No | `false` | `true` sends [`eager_end_of_turn`](#eager_end_of_turn) in `auto` mode. | +| `turn_detection.silence_timeout_ms` | No | Depends on mode | See [`silence_timeout_ms`](#silence_timeout_ms). | +| `results.drafts` | No | `false` | `true` sends live [`draft`](#draft) messages. | +| `results.draft_interval_ms` | No | `250` | Minimum time between drafts. Smaller values count as 250. | +| `results.ignore_timestamps` | No | `true` | `false` adds word timings (`words`) to `committed`. | +| `results.diarize` | No | `true` | `true`, `false` or `"auto"` (same as `true`); the strings `"true"` and `"false"` are accepted too. `false` removes speaker tags from the text. See [Speakers](#speakers). | +| `results.vad_events` | No | `false` | `true` sends [`vad`](#vad) messages. | +| `results.eot_events` | No | `false` | `true` sends [`eot_prob`](#eot_prob) messages. | +| `context.history` | No | `[]` | Earlier conversation, oldest first: up to 32 items of `{ "role": "user" \| "assistant", "text": str }`, each up to 2,000 characters. | +| `context.terms` | No | `[]` | Up to 100 terms of up to 32 characters each: names, products and jargon to recognize. | +| `client_reference_id` | No | | Your own ID for the session, up to 256 characters. It is recorded with the session and not sent back. | + +A value of the wrong type for `ignore_timestamps`, or a `diarize` value other than those listed, is `invalid_config`. + +#### Audio format + +Send 16-bit little-endian PCM, mono, at one of the listed sample rates. Other encodings (such as μ-law, A-law or Opus) and stereo audio are `unsupported_audio`; decode phone audio in μ-law or A-law to 16-bit PCM first, and send it at 8000 Hz as is. Audio at 16000 Hz is used as is; every other rate is converted on the server, so you never need to resample. The conversion adds at most about 20 ms of delay. Every `*_ms` value is in milliseconds of your audio, whatever its rate. + +An `audio` message can hold any number of bytes; an odd trailing byte waits for the next message. Send 80–100 ms of audio per message. A message can be at most 1 MiB, which holds 65.5 seconds of audio at 8 kHz or 10.9 seconds at 48 kHz. + +#### Turn detection modes + +| Mode | A turn ends when | Use it for | +| -------- | --------------------------------------------------------------------------------------------- | -------------------------------------------- | +| `auto` | The server's turn detection judges that the speaker has finished, from what was said and how. | Voice agents and conversations. The default. | +| `vad` | `silence_timeout_ms` of silence has passed (default 700 ms). | Fixed silence-based endpointing. | +| `manual` | You send `finalize`. | Push-to-talk, or your own endpointing. | + +**`auto`.** A turn opens as soon as voice activity is detected. The server then judges whether this is the user taking their turn. If it is, the server sends `user_turn_taken`, and it ends the turn when the speaker has finished, not on silence alone. A short pause in the middle of a sentence usually does not end the turn; a speaker who stops mid-sentence and does not go on has the turn ended about 2 seconds after they stopped. `silence_timeout_ms` is only a safety net here. + +Other sounds, usually noise, a TV or other people talking nearby, end after a short quiet, or after 20 seconds at the latest, and are committed as non-user turns with `user_turn: false`. When the user starts speaking during such a sound, the sound before the speech is usually committed first as a non-user turn, and the speech opens a new turn. A sound that began shortly before the speech stays in the user's turn. A turn you end with `finalize` or `audio_done` before `user_turn_taken` is not split: all of it becomes one user turn. + +This judgement needs `auto` in the **first** `config`; a `config` without `mode` has it. A session that starts in `vad` or `manual` and switches to `auto` later makes every turn a user turn as it opens (`user_turn_taken` right after `speech_start`, never `user_turn: false`). A session that starts in `auto` keeps the judgement when it switches away and back. In rare cases every turn of a session is treated as a user turn; `user_turn_taken` then follows `speech_start` immediately. + +**`vad`.** A turn ends after `silence_timeout_ms` of silence, with `reason: "eot"`. Every turn is a user turn. + +**`manual`.** A turn ends when you send `finalize`. A turn that has been silent for `silence_timeout_ms` ends too; at the default (60000) that never happens, because the 60-second turn limit comes first. Every turn is a user turn. + +In every mode, a turn ends after 60 seconds at the latest, and an `auto` turn without `user_turn_taken` after 20 seconds (`reason: "max_turn_duration"`). + +#### `silence_timeout_ms` + +The default, the allowed range (inclusive) and the meaning depend on the mode: + +| Mode | Default | Allowed | Meaning | +| -------- | ------- | ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `auto` | 5000 | 3000–60000 | A safety net for user turns: a user turn with no voice activity for this long ends with `reason: "silence_timeout"`. Turn detection normally ends a turn long before that. Non-user turns do not use it. | +| `vad` | 700 | 300–60000 | The silence that ends a turn (`reason: "eot"`). | +| `manual` | 60000 | 300–60000 | A safety net: a turn silent for this long ends with `reason: "silence_timeout"`. At 60000 it never fires. | + +- If you leave it out, the mode's default applies. `session_started` and `configured` always show the value in effect. +- The value must be a whole number inside the range of the mode it will run under: the `mode` in the same message, or else the current mode. In `config` the current mode is `auto`, so a `config` with `silence_timeout_ms: 700` and no `mode` is refused. +- Outside the range, `config` gets `error` with `invalid_config` and the connection closes with 4400; the message gives the range, for example "turn_detection.silence_timeout_ms must be between 3000 and 60000 in auto mode". A `configure` gets `configured` with `ok: false` and the same message, and nothing in that `configure` is applied. +- A value that is not a non-negative integer is `invalid_config` in `config`, and a recoverable `invalid_request` (nothing applied) in `configure`. +- A `configure` that switches `mode` without `silence_timeout_ms` resets it to the new mode's default. To switch and keep your own value, send both. +- A new value takes effect at once, for the open turn too, from the point in the audio where the `configure` arrives. + +### `audio` + +```json +{ "type": "audio", "data": "" } +``` + +Audio in the format from `config`, as MessagePack `bin`. Send audio only after `session_started`. For live audio, send it as it is captured; 80–100 ms per message is a good size. You can send faster than real time, for example from a file: the server accepts it and reads it at a bounded rate, and results keep following the audio. + +### `finalize` + +```json +{ "type": "finalize" } +``` + +Commit the current turn now, with `reason: "finalize"`. In `auto` mode, the server first settles its turn decisions on the audio already sent: a turn they already ended (possibly with `user_turn: false`) is not reopened, and if no turn is open after that, `finalize` is ignored. The turn `finalize` ends is always a user turn; `user_turn_taken` is sent first if it has not been. + +**Push-to-talk:** use `mode: "manual"` and send `finalize` when the button is released. Send audio while the button is held; a turn opens on voice activity, so if none was detected, no turn is open and `finalize` is ignored. While the button is up you can send no audio, which keeps other sounds from opening turns; send `keepalive` then. + +### `configure` + +```json +{ + "type": "configure", + "turn_detection": { "mode": "vad", "silence_timeout_ms": 900 }, + "results": { "drafts": false }, + "context": { "terms": ["Fish Audio", "Main Street"] } +} +``` + +Change settings during the session. Every field is optional and takes the same values as in `config`; a field you leave out keeps its current value. `context` accepts `terms` only, and the new list replaces the old one; `context.history` is ignored in `configure`, so send earlier lines with [`context_add`](#context_add). The server answers with [`configured`](#configured). If any value is invalid, nothing in the message is applied. + +- A change to `ignore_timestamps` or `diarize` applies to turns that open after `configured`. A turn that is already open keeps the settings it started with, so its `eager_end_of_turn` and `committed` always agree. +- When `mode` changes to or from `auto`, the server first settles its turn decisions on the audio already sent, so `configured` can take a few seconds. +- A turn's `user_turn` status follows the mode it opened in: a turn opened in `vad` or `manual` mode is always a user turn and gets no `user_turn_taken`. How an open turn ends follows the current mode. +- Switching away from `auto` settles an open turn that has no `user_turn_taken` yet: it becomes a user turn (`user_turn_taken` is sent) if there was speech in about the last second; otherwise it is committed at once with `user_turn: false`. + +### `context_add` + +```json +{ + "type": "context_add", + "items": [{ "role": "assistant", "text": "Sure. Which day works for you?" }], + "reset_history": false +} +``` + +Add conversation context. The server keeps the session's conversation and uses the recent part of it to recognize names and follow-ups. Every committed user turn is added as a `user` item automatically; non-user turns are not added. Send what the other party said, for a voice agent each line it speaks, as `assistant` items, ideally right after the `committed` it answers. Items follow the `context.history` limits (up to 32 items of up to 2,000 characters per message). `reset_history: true` first drops the items you added before. There is no reply. + +### `keepalive` + +```json +{ "type": "keepalive" } +``` + +Keeps an idle session open. A session closes after 30 seconds without any client message. + +### `audio_done` + +```json +{ "type": "audio_done" } +``` + +No more audio. The server commits the open turn (`reason: "audio_done"`), sends `session_done`, and closes the connection with 1000. In `auto` mode it first settles its turn decisions on the audio already sent; the turn `audio_done` ends is always a user turn, as with `finalize`. + +### `agent_speaking` + +```json +{ "type": "agent_speaking", "speaking": true } +``` + +For voice agents in `auto` mode: tell the server when your agent's reply is in progress, so it can tell a short acknowledgement from the user taking the turn. While the agent is speaking, a "mm-hm" or "yeah" from the user does not take the turn: the server sends [`user_backchannel`](#user_backchannel) instead of `user_turn_taken`, and the turn commits as a non-user turn with `turn_kind: "backchannel"`. + +- **When to send `true`:** when you commit to playing a reply, typically on `committed`, or, for a reply you prepared on `eager_end_of_turn`, when you start playing it. Preparing a reply is not speaking: do not send `true` on `eager_end_of_turn` itself, since the turn can still resume. "Speaking" covers the wait for your speech synthesis and the gaps between sentences. +- **When to send `false`:** when the last audio sample of the reply has played to the user, or when you stop the reply (barge-in). If you start playing a reply before `committed` and `turn_resumed` then cancels it, stop it and send `false`. +- **Pauses:** if a reply pauses for more than about 2 seconds, send `false` for the pause and `true` again when it goes on. +- **Default:** not speaking. Without this message, every acknowledgement can take the turn. Send it only when the value changes; repeating it changes nothing. +- **Timing:** it applies at the point in the audio where it arrives: after the audio you sent before it. Send it on the same connection as your audio, at the moment the reply starts or stops. +- **Other modes:** in `vad` and `manual` mode it is accepted and remembered, and has no effect. After a switch to `auto`, the remembered value applies. +- `speaking` must be a boolean. Anything else, or a message without it, is a recoverable `invalid_request` and changes nothing. +- Keep acoustic echo cancellation on. Without it, the agent's own voice can reach the microphone, open turns, and be taken for the user. + +## Server messages + +`draft`, `committed` and `progress` carry three watermarks, in milliseconds of audio: `audio_received_ms` (audio the server has received), `audio_processed_ms` (audio it has processed, never more than it received) and `audio_committed_ms` (the end of the audio covered by committed turns). The difference between the first two shows how far processing is behind your audio. + +### `session_started` + +| Field | Type | Description | +| ------------ | ------ | --------------------------------------------------------------------------------- | +| `session_id` | string | The session's ID. | +| `request_id` | string | The same value as the `x-request-id` header. Include it when you contact support. | +| `config` | map | The effective configuration, with defaults filled in. | +| `limits` | map | `max_session_ms`, `max_turn_ms` and `max_frame_bytes`. See [Limits](#limits). | + +Send audio after this message. + +### `speech_start` + +| Field | Type | Description | +| ------------ | ------- | -------------------------------------------------- | +| `turn_index` | integer | The turn that opened, counting from 0. | +| `audio_ms` | integer | Where the turn starts, slightly before the speech. | + +A turn opened: voice activity began. It is sent shortly after voice activity begins. In `auto` mode, noise and other voices open turns too, so wait for `user_turn_taken` before treating it as the user's turn. In `auto` mode it can come later if you stream faster than real time or in messages of about 0.3 seconds or more; the turns are the same as for a real-time stream. If the speaker is still talking when a turn ends, the next turn starts at that turn's `audio_end_ms`. + +### `user_turn_taken` + +| Field | Type | Description | +| ------------ | ------- | ---------------------------------------------------- | +| `turn_index` | integer | The turn. | +| `audio_ms` | integer | Where the server judged that the user took the turn. | + +`auto` mode only: the server judged that the user is taking their turn. It usually comes within the first second or so of speech, and at the latest right before `committed` (always for a turn ended by `finalize` or `audio_done`). At most once per turn. When the sound before the user's speech is committed on its own, this turn's `speech_start` and `user_turn_taken` come right after that `committed`. + +### `draft` + +| Field | Type | Description | +| --------------- | ------- | ----------------------------------------------------------------- | +| `turn_index` | integer | The turn. | +| `text` | string | The whole turn so far. Not final. | +| `language` | string | Optional. The language's English name. See [Language](#language). | +| `language_code` | string | Optional. ISO 639-1 code. See [Language](#language). | +| `audio_ms` | integer | The audio position the draft covers up to. | + +Each draft **replaces** the previous one for the turn. Drafts are best effort, sent at most every `draft_interval_ms`, and only with `results.drafts: true`. Turn them on for live captions; an agent that acts on `eager_end_of_turn` and `committed` does not need them. Drafts may pause during long background sounds and resume when `user_turn_taken` comes. + +### `eager_end_of_turn` + +| Field | Type | Description | +| --------------- | ------- | ------------------------------------------- | +| `turn_index` | integer | The turn. | +| `text` | string | The turn's text. Can be empty. | +| `language` | string | Optional. See [Language](#language). | +| `language_code` | string | Optional. See [Language](#language). | +| `audio_ms` | integer | Optional. The audio position of the signal. | + +With `turn_detection.eager: true` in `auto` mode, and only after `user_turn_taken`: the turn has probably ended. `text` and the language are exactly what `committed` will say unless `turn_resumed` follows, so you can start preparing your response early. Not every user turn has one. It usually comes some time before `committed`, and sometimes right before it. + +### `turn_resumed` + +| Field | Type | Description | +| ------------ | ------- | ----------- | +| `turn_index` | integer | The turn. | + +The speaker went on after `eager_end_of_turn`: discard that text. If you already started playing a reply to it and sent `agent_speaking: true`, stop the reply and send `agent_speaking: false`. + +### `user_backchannel` + +| Field | Type | Description | +| ------------ | ------- | ------------------------------------------------------------------------ | +| `turn_index` | integer | The open turn, or, if no turn is open, the index the next turn will get. | +| `audio_ms` | integer | Where the acknowledgement is in the audio. | + +`auto` mode only, while your agent is speaking (see [`agent_speaking`](#agent_speaking)): the user acknowledged ("mm-hm", "yeah", "right") without taking the turn. Keep talking. One message per acknowledgement. It always comes before the named turn's `committed`, and it can come before that turn's `speech_start`. A backchannel counts for the turn that contains it; that turn commits with `user_turn: false` and `turn_kind: "backchannel"` unless the user goes on to take the turn. A turn that opened in `vad` or `manual` mode never counts an acknowledgement. When your agent is not speaking, an acknowledgement takes the turn (`user_turn_taken`). + +### `committed` + +| Field | Type | Description | +| ---------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `turn_index` | integer | The turn. | +| `text` | string | The final transcript of the turn. Can be empty. | +| `words` | array | Present when `ignore_timestamps` is `false`: `[{ "text", "start_ms", "end_ms" }]`, or `[]` when there is nothing to align. | +| `language` | string | Optional. See [Language](#language). | +| `language_code` | string | Optional. See [Language](#language). | +| `audio_start_ms` | integer | Where the turn starts. | +| `audio_end_ms` | integer | Where the turn ends. | +| `reason` | string | Why the turn ended: `eot`, `finalize`, `silence_timeout`, `max_turn_duration` or `audio_done`. See [Why a turn ended](#why-a-turn-ended). | +| `degraded` | boolean | `true` when the final pass failed. See [Degraded commits](#degraded-commits). | +| `user_turn` | boolean | `true` for the user's turn; `false` for a non-user turn. Always `true` in `vad` and `manual` mode. If it is ever missing, treat it as `true`. | +| `turn_kind` | string | `"user"` (exactly when `user_turn` is true), `"backchannel"` (a non-user turn holding an acknowledgement) or `"other"` (noise, another voice, unclear). | + +The final transcript of the turn. Decide whether to answer a turn by `user_turn`, never by `reason`: a turn the server ends itself (for example on `server_restarting`) has `reason: "finalize"` and may have `user_turn: false`. Treat a `turn_kind` you do not recognize like `"other"` when `user_turn` is false. A non-user turn is usually noise or another voice, often with empty `text`. + +### `vad` + +| Field | Type | Description | +| ---------- | ------- | ---------------------------- | +| `audio_ms` | integer | The audio position. | +| `score` | number | Voice activity, from 0 to 1. | + +Up to 10 per second, with `results.vad_events: true`. + +### `eot_prob` + +| Field | Type | Description | +| ------------ | ------- | ------------------------------------ | +| `turn_index` | integer | The turn. | +| `audio_ms` | integer | The audio position. | +| `prob` | number | End-of-turn likelihood, from 0 to 1. | + +With `results.eot_events: true`, for display only. In `vad` mode it is 0.7 × the silence so far ÷ `silence_timeout_ms`: the turn ends where it reaches 0.7. + +### `configured` + +| Field | Type | Description | +| --------- | ------- | ---------------------------------------------------------- | +| `ok` | boolean | `false` means nothing changed. | +| `config` | map | The configuration in effect after the `configure`. | +| `message` | string | Present when `ok` is `false`: why, in English, for humans. | + +The reply to `configure`. `ok: false` comes for an invalid value, such as a `silence_timeout_ms` outside the mode's range, or a value over a limit. + +### `progress` + +Watermarks only. Sent every 5 seconds while the client sends nothing. + +### `error` + +| Field | Type | Description | +| ------------- | ------- | ------------------------------------------------------------------------- | +| `code` | integer | An HTTP-style status: 400, 408, 413, 500, 503 or 504. | +| `error_type` | string | What went wrong. See [Errors and close codes](#errors-and-close-codes). | +| `message` | string | A short English description, for humans. Do not parse it. | +| `request_id` | string | The session's request ID. | +| `recoverable` | boolean | `true` when the session continues, except `server_restarting`: see below. | + +### `session_done` + +| Field | Type | Description | +| ------- | ---- | ----------------------------------------------------------------------------------------------- | +| `stats` | map | `audio_ms` (audio received), `turns` (every committed turn, user turn or not) and `request_id`. | + +Sent after `audio_done`, just before the server closes the connection with 1000. + +## Guarantees + +- Every turn has exactly one `speech_start` and one `committed`, and nothing about that turn follows its `committed`. +- `user_turn_taken` comes at most once per turn, between `speech_start` and `committed`, and before any `eager_end_of_turn`. For a turn opened in `auto` mode, `user_turn` is true exactly when `user_turn_taken` was sent. +- A turn has at most one `eager_end_of_turn` that no `turn_resumed` follows. If it has one, `committed.text` and `committed.language_code` equal its own. +- `user_turn` is true exactly when `turn_kind` is `"user"`. +- `user_backchannel` comes only in `auto` mode, while your agent is speaking, and before its turn's `committed`. +- `language` and `language_code` are both present or both absent. +- `committed.words` is present exactly when `ignore_timestamps` was `false` when the turn opened and the commit is not degraded. + +## Why a turn ended + +`committed.reason` depends on the mode: + +| Mode | `eot` | `silence_timeout` | +| -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | +| `auto` | Turn detection ended the turn: the speaker finished, or stopped mid-sentence and did not go on (about 2 seconds later). Non-user turns, such as a sound that went quiet, end with `eot` too. | The safety net ended a user turn (`silence_timeout_ms`, default 5000). | +| `vad` | `silence_timeout_ms` of silence (default 700). | Never. | +| `manual` | Never. | `silence_timeout_ms` of silence (never at the default 60000). | + +In every mode: + +- `finalize`: you sent `finalize`, or the server ended the session (`server_restarting`, `max_duration_exceeded`). +- `max_turn_duration`: the turn reached 60 seconds, or 20 seconds for an `auto` turn without `user_turn_taken`. +- `audio_done`: you sent `audio_done`. + +**Treat a `reason` you do not recognize like `eot`.** To decide whether to answer a turn, use `user_turn`, not `reason`. + +## Speakers + +`text` can contain speaker tags `<|speaker:N|>` (N = 0, 1, ...). Numbers are stable within a session and start over in a new one. Strip the tags if you only need the words, or send `results.diarize: false` to get text without them. With `diarize: false`, whitespace in the text, line breaks included, is collapsed to single spaces and trimmed; sound tags in square brackets, such as `[laughter]`, stay. + +## Language + +`draft`, `eager_end_of_turn` and `committed` carry the language of their own `text`. Use `language_code` (ISO 639-1) in your code; `language` is its English name, for display, with the same names as [`POST /v1/asr`](/api-reference/endpoint/openapi-v1/speech-to-text). The language is detected from that message's text alone, ignoring speaker tags, sound tags and interjections such as "okay" or "hmm". It is one of: + +`ar`, `zh`, `cs`, `da`, `nl`, `en`, `fi`, `fr`, `de`, `el`, `hi`, `hu`, `id`, `it`, `ja`, `ko`, `mk`, `ms`, `fa`, `pl`, `pt`, `ro`, `ru`, `es`, `sv`, `tl`, `th`, `tr`, `vi`. + +Both fields are omitted when the text is empty, too short to tell (most replies under three words, such as "Okay" or "Yes.") or ambiguous; keep the last label you saw. Mixed-language text gets one label, the dominant language. Drafts are labeled one by one and may change; `committed` is final. + +## Degraded commits + +If the final pass for a turn fails, you receive `error` with `engine_error` and `recoverable: true`, and then a `committed` with `degraded: true`. Its `text` and language are those of the turn's last draft (empty and without a language when the turn had no draft, as with `results.drafts: false`), and it has no `words`. The session continues. + +## Errors and close codes + +| `error_type` | `code` | Recoverable | Then | +| ----------------------- | --------------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `invalid_config` | 400 | No | Close 4400. A bad or missing `config`, a value over a limit, or `silence_timeout_ms` outside the mode's range. | +| `unsupported_audio` | 400 | No | Close 4400. A sample rate not in the list, a format other than `pcm_s16le`, or more than one channel. | +| `invalid_request` | 400 | Yes | The session continues. A text frame, an unreadable message, an over-limit `context_add`, a second `config`, or `agent_speaking` without a boolean `speaking`. | +| `idle_timeout` | 408 | No | Close 4408 after 30 seconds without any client message. | +| `max_duration_exceeded` | 413 | No | The open turn is committed (`reason: "finalize"`), then close 4413. | +| `frame_too_large` | 413 | No | Close 1009. A message over `limits.max_frame_bytes`. | +| `engine_error` | 500, 503 or 504 | Yes | A [degraded `committed`](#degraded-commits) follows (503: temporarily unavailable; 504: timed out). If the session could not start: `recoverable: false`, close 4500. | +| `internal` | 500 | No | Close 4500. | +| `server_restarting` | 503 | Yes | The open turn is committed (`reason: "finalize"`), then close 1012. Reconnect for a new session. | + +`server_restarting` is the one recoverable error that closes the connection: the session cannot go on, but you can reconnect at once for a new one. If the first message is not a readable `config`, you get an `error` with `recoverable: false` and close 4400. + +A normal end is `session_done`, then close 1000. Match on `error_type`, not on `message`. In `auto` mode, a turn the server commits before closing may have `user_turn: false`. + +## Limits + +| Limit | Value | +| -------------------- | ------------------------------------------------------------------------------------------ | +| Audio | `pcm_s16le`, mono, at 8000, 12000, 16000, 22050, 24000, 32000, 44100 or 48000 Hz | +| Session length | 3,600 seconds (`limits.max_session_ms`) | +| Turn length | 60 seconds (`limits.max_turn_ms`); 20 seconds for an `auto` turn without `user_turn_taken` | +| `silence_timeout_ms` | `auto`: 3000–60000; `vad` and `manual`: 300–60000 | +| Message size | 1 MiB (`limits.max_frame_bytes`) | +| First `config` | Within 10 seconds of connecting | +| Idle | 30 seconds without any client message | +| Drafts | At most every 250 ms | +| Context | 32 items × 2,000 characters per message; 100 terms × 32 characters | + +## Example + +A session in `auto` mode with eager end of turn, drafts and word timings. `C` is the client and `S` the server; fields are abbreviated. + +```text +C → config {audio: {format: "pcm_s16le", sample_rate: 16000, channels: 1}, + turn_detection: {mode: "auto", eager: true}, + results: {drafts: true, ignore_timestamps: false}} +S → session_started {seq: 0, session_id: "s_ab12…", request_id: "…", config: {…}, limits: {…}} +C → audio {data: } … +S → speech_start {seq: 1, turn_index: 0, audio_ms: 300} +S → draft {seq: 2, turn_index: 0, text: "what's the", audio_ms: 900, …} +S → user_turn_taken {seq: 3, turn_index: 0, audio_ms: 1280} +S → draft {seq: 4, turn_index: 0, text: "what's the weather like", language: "English", + language_code: "en", audio_ms: 1800, …} +S → eager_end_of_turn {seq: 5, turn_index: 0, text: "What's the weather like?", + language: "English", language_code: "en", audio_ms: 2240} +S → committed {seq: 6, turn_index: 0, text: "What's the weather like?", words: […], + language: "English", language_code: "en", audio_start_ms: 300, + audio_end_ms: 2880, reason: "eot", degraded: false, user_turn: true, + turn_kind: "user", …} +C → agent_speaking {speaking: true} +C → context_add {items: [{role: "assistant", text: "Sunny, 24 degrees, and a light breeze."}]} +S → speech_start {seq: 7, turn_index: 1, audio_ms: 3520} +S → user_backchannel {seq: 8, turn_index: 1, audio_ms: 3840} +S → committed {seq: 9, turn_index: 1, text: "Mm-hm.", audio_start_ms: 3520, audio_end_ms: 6080, + reason: "eot", degraded: false, user_turn: false, turn_kind: "backchannel", …} +C → agent_speaking {speaking: false} +S → speech_start {seq: 10, turn_index: 2, audio_ms: 7040} +S → committed {seq: 11, turn_index: 2, text: "", audio_start_ms: 7040, audio_end_ms: 8640, + reason: "eot", degraded: false, user_turn: false, turn_kind: "other", …} +C → audio_done +S → session_done {seq: 12, stats: {audio_ms: 8960, turns: 3, request_id: "…"}} + close 1000 +``` + +Turn 0 is the user's turn. The agent prepared its reply on `eager_end_of_turn`; once the turn commits, it starts playing the reply and sends `agent_speaking: true`. The first draft has no language: "what's the" is too short to tell. Turn 1 is the user saying "mm-hm" while the agent speaks: the server sends `user_backchannel`, so the agent keeps talking, and the turn commits as a non-user turn that is not added to the context. When the reply has played out, the agent sends `agent_speaking: false`. Turn 2 is a non-user turn, a sound such as a cough: the agent ignores it. + +## Compatibility rules + +- **Ignore fields and message types you do not know.** Version 1 only ever adds them. Breaking changes get a new path (`/v2/...`). +- Treat an unknown `reason` like `eot`, and an unknown `turn_kind` like `"other"` when `user_turn` is `false`. +- Match errors on `error_type`, not on `message`. + +## Related + +- [Realtime Speech to Text](/features/speech-to-text/realtime): what the realtime model does and when to use it. +- [Transcribe a live audio stream](/developer-guide/sdk-guide/cookbook/realtime-transcription): a complete client in Python and Node.js, and a browser microphone relay. +- [Build a realtime voice agent](/developer-guide/sdk-guide/cookbook/realtime-voice-agent): turn-taking, barge-in and backchannels. +- [Speech to Text](/api-reference/endpoint/openapi-v1/speech-to-text): transcribe a complete audio file with `POST /v1/asr`. diff --git a/developer-guide/sdk-guide/cookbook/realtime-transcription.mdx b/developer-guide/sdk-guide/cookbook/realtime-transcription.mdx new file mode 100644 index 00000000..f3a40fa5 --- /dev/null +++ b/developer-guide/sdk-guide/cookbook/realtime-transcription.mdx @@ -0,0 +1,498 @@ +--- +title: "Transcribe a live audio stream in realtime" +description: "Stream audio to transcribe-1-pro-realtime over a WebSocket and print drafts, turn events and final transcripts as they arrive" +icon: "tower-broadcast" +hidden: true +noindex: true +--- + +import Prerequisites from "/snippets/prerequisites.mdx"; + + + Realtime speech to text is in preview and is enabled per team. Contact + [support@fish.audio](mailto:support@fish.audio) to turn it on for your team. + Until then, connecting returns `403`. + + +## Prerequisites + + + +The SDKs do not wrap the realtime speech-to-text endpoint yet, so this recipe speaks the WebSocket protocol directly. Install a WebSocket client and a MessagePack library: + + + +```bash Python +pip install "websockets>=14" msgpack +``` + +```bash JavaScript +npm install ws @msgpack/msgpack +``` + + + +You also need a 16-bit PCM mono WAV file, at 8000, 12000, 16000, 22050, 24000, 32000, 44100 or 48000 Hz. To convert a recording, run `ffmpeg -i input.mp3 -ac 1 -ar 16000 -c:a pcm_s16le speech.wav`. + +## Recipe + +The script connects to `wss://api.fish.audio/v1/asr/realtime`, sends a `config` message, waits for `session_started`, and then streams the file in 100 ms messages at real-time speed, as a live microphone would. It prints every message the server sends until the session ends. + + + +```python Python +"""Stream a WAV file to Fish Audio realtime speech-to-text and print the events. + + pip install "websockets>=14" msgpack + export FISH_API_KEY=... + python transcribe_file.py speech.wav + +The WAV must be 16-bit PCM, mono, at 8000, 12000, 16000, 22050, 24000, 32000, 44100 or 48000 Hz. +""" +import asyncio +import os +import sys +import wave + +import msgpack +import websockets + +URL = "wss://api.fish.audio/v1/asr/realtime" +CHUNK_MS = 100 # 80-100 ms of audio per message + + +def pack(message): + return msgpack.packb(message) # bytes values are sent as MessagePack bin + + +async def stream_file(ws, wav, rate): + """Send the audio in real time, as a live microphone would, then audio_done.""" + loop = asyncio.get_running_loop() + start = loop.time() + sent_ms = 0 + while chunk := wav.readframes(rate * CHUNK_MS // 1000): + await ws.send(pack({"type": "audio", "data": chunk})) + sent_ms += CHUNK_MS + await asyncio.sleep(max(0.0, start + sent_ms / 1000 - loop.time())) + await ws.send(pack({"type": "audio_done"})) + + +async def main(path): + wav = wave.open(path, "rb") + if wav.getsampwidth() != 2 or wav.getnchannels() != 1: + sys.exit("need a 16-bit mono WAV") + rate = wav.getframerate() + headers = {"Authorization": f"Bearer {os.environ['FISH_API_KEY']}"} + + try: + ws = await websockets.connect(URL, additional_headers=headers) + except websockets.InvalidStatus as e: # the upgrade was refused: 401, 402, 403, 429, 503 ... + sys.exit(f"refused: HTTP {e.response.status_code} {e.response.body.decode(errors='replace')}") + + async with ws: + await ws.send(pack({ + "type": "config", + "audio": {"format": "pcm_s16le", "sample_rate": rate, "channels": 1}, + "turn_detection": {"mode": "auto", "eager": True}, + "results": {"drafts": True}, + })) + sender = None + try: + async for frame in ws: + m = msgpack.unpackb(frame) + t = m["type"] + if t == "session_started": + print(f"session {m['session_id']} (request {m['request_id']})") + sender = asyncio.create_task(stream_file(ws, wav, rate)) # send audio after this + elif t == "draft": + print(f" draft #{m['turn_index']}: {m['text']}") + elif t == "eager_end_of_turn": + print(f" eager #{m['turn_index']}: {m['text']}") + elif t == "committed": + who = "user " if m.get("user_turn", True) else "other" + print(f"committed #{m['turn_index']} [{who}] {m['audio_start_ms']}-{m['audio_end_ms']} ms " + f"({m['reason']}, {m.get('language_code', '?')}): {m['text']}") + elif t == "error": + print(f"error {m['error_type']} ({m['code']}): {m['message']} [request {m['request_id']}]") + elif t == "session_done": + print(f"done: {m['stats']['audio_ms']} ms of audio, {m['stats']['turns']} turns") + else: # speech_start, user_turn_taken, turn_resumed, ... (ignore types you don't know) + print(f" {t} #{m.get('turn_index', '')}") + except websockets.ConnectionClosedError: + pass + finally: + if sender: + sender.cancel() + print(f"closed: {ws.close_code} {ws.close_reason}") + + +if __name__ == "__main__": + asyncio.run(main(sys.argv[1])) +``` + +```javascript JavaScript +// npm install ws @msgpack/msgpack +// FISH_API_KEY=... node transcribe_file.mjs speech.wav +import { readFileSync } from "node:fs"; +import { setTimeout as sleep } from "node:timers/promises"; +import { encode, decode } from "@msgpack/msgpack"; +import WebSocket from "ws"; + +const URL = "wss://api.fish.audio/v1/asr/realtime"; +const CHUNK_MS = 100; // 80-100 ms of audio per message + +// Read a 16-bit mono PCM WAV file: its sample rate and its samples. +function readWav(path) { + const buf = readFileSync(path); + let rate, channels, bits, data; + for (let pos = 12; pos + 8 <= buf.length; ) { + const id = buf.toString("ascii", pos, pos + 4); + const size = buf.readUInt32LE(pos + 4); + if (id === "fmt ") { + channels = buf.readUInt16LE(pos + 10); + rate = buf.readUInt32LE(pos + 12); + bits = buf.readUInt16LE(pos + 22); + } else if (id === "data") { + data = buf.subarray(pos + 8, pos + 8 + size); + } + pos += 8 + size + (size % 2); + } + if (bits !== 16 || channels !== 1 || !data) + throw new Error("need a 16-bit mono WAV"); + return { rate, data }; +} + +const { rate, data } = readWav(process.argv[2]); +const ws = new WebSocket(URL, { + headers: { Authorization: `Bearer ${process.env.FISH_API_KEY}` }, +}); +const send = message => ws.send(encode(message)); // Buffers are sent as MessagePack bin + +// Send the audio in real time, as a live microphone would, then audio_done. +async function streamFile() { + const bytesPerChunk = ((rate * CHUNK_MS) / 1000) * 2; + const start = Date.now(); + for (let i = 0; i < data.length; i += bytesPerChunk) { + if (ws.readyState !== WebSocket.OPEN) return; + send({ type: "audio", data: data.subarray(i, i + bytesPerChunk) }); + const sentMs = (i / bytesPerChunk + 1) * CHUNK_MS; + await sleep(Math.max(0, start + sentMs - Date.now())); + } + send({ type: "audio_done" }); +} + +ws.on("unexpected-response", (req, res) => { + // The upgrade was refused: 401, 402, 403, 429, 503 ... + let body = ""; + res.on("data", chunk => (body += chunk)); + res.on("end", () => { + console.error(`refused: HTTP ${res.statusCode} ${body}`); + process.exit(1); + }); +}); + +ws.on("open", () => + send({ + type: "config", + audio: { format: "pcm_s16le", sample_rate: rate, channels: 1 }, + turn_detection: { mode: "auto", eager: true }, + results: { drafts: true }, + }) +); + +ws.on("message", frame => { + const m = decode(frame); + switch (m.type) { + case "session_started": + console.log(`session ${m.session_id} (request ${m.request_id})`); + streamFile(); // send audio after this + break; + case "draft": + console.log(` draft #${m.turn_index}: ${m.text}`); + break; + case "eager_end_of_turn": + console.log(` eager #${m.turn_index}: ${m.text}`); + break; + case "committed": { + const who = m.user_turn === false ? "other" : "user "; + console.log( + `committed #${m.turn_index} [${who}] ${m.audio_start_ms}-${m.audio_end_ms} ms ` + + `(${m.reason}, ${m.language_code ?? "?"}): ${m.text}` + ); + break; + } + case "error": + console.log( + `error ${m.error_type} (${m.code}): ${m.message} [request ${m.request_id}]` + ); + break; + case "session_done": + console.log( + `done: ${m.stats.audio_ms} ms of audio, ${m.stats.turns} turns` + ); + break; + default: // speech_start, user_turn_taken, turn_resumed, ... (ignore types you don't know) + console.log(` ${m.type} #${m.turn_index ?? ""}`); + } +}); + +ws.on("close", (code, reason) => console.log(`closed: ${code} ${reason}`)); +``` + + + +Run it with your API key in `FISH_API_KEY`: + +```bash +python transcribe_file.py speech.wav +# or +node transcribe_file.mjs speech.wav +``` + +For a 15-second customer-support clip, the output looks like this (`...` marks drafts left out): + +```text +session s_ab12cd34 (request 0b6f4c1e-7d2a-4e8b-9f3c-5a1d2e3f4b6c) + speech_start #0 + draft #0: <|speaker:0|>Can you... + draft #0: <|speaker:0|>Can you tell + user_turn_taken #0 + draft #0: <|speaker:0|>Can you tell me what... + ... + draft #0: <|speaker:0|>Can you tell me what time the pharmacy on Main Street closes tonight? + eager #0: <|speaker:0|>Can you tell me what time the pharmacy on Main Street closes tonight? +committed #0 [user ] 2768-8000 ms (eot, en): <|speaker:0|>Can you tell me what time the pharmacy on Main Street closes tonight? + speech_start #1 + draft #1: <|speaker:0|>Uh-huh. + user_turn_taken #1 + eager #1: <|speaker:0|>Uh-huh. +committed #1 [user ] 9840-11840 ms (eot, ?): <|speaker:0|>Uh-huh. + speech_start #2 + draft #2: <|speaker:0|>I'd like + user_turn_taken #2 + ... + draft #2: <|speaker:0|>I'd like to change my deli- +committed #2 [user ] 13168-14992 ms (audio_done, en): <|speaker:0|>I'd like to change my delivery +done: 15000 ms of audio, 3 turns +closed: 1000 done +``` + +The clip ends mid-sentence, so the last turn is committed by `audio_done`. Drafts can hold partial words, such as `deli-`; the committed text is final. + +## Read the events + +- **`speech_start`** opens a turn when voice activity begins. In the default `auto` mode, noise and other voices open turns too. +- **`draft`** is the whole turn so far, and replaces the previous draft. Drafts come only with `results.drafts: true`; use them for live captions. +- **`user_turn_taken`** says the server judged that the user is taking their turn, rather than noise or someone else nearby. +- **`eager_end_of_turn`** comes with `turn_detection.eager: true` when the turn has probably ended. Its text equals the final transcript unless `turn_resumed` follows, so an agent can start its reply early. +- **`committed`** is the final transcript of the turn. `user_turn: false` marks a non-user turn, such as background noise or a short acknowledgement while your agent is speaking. Speaker tags such as `<|speaker:0|>` are stable within the session; strip them, or send `results.diarize: false`. `language_code` is omitted when the text is too short to tell, as for "Uh-huh." above. +- **`session_done`** follows `audio_done`, and the server then closes the connection with code 1000. + +Ignore message types and fields you do not know: new ones can be added. Every message is described in the [WebSocket reference](/api-reference/endpoint/websocket/asr-realtime#server-messages). + + + The script streams the file at real-time speed so that the events arrive as + they would for live audio. To transcribe a recording you already have, use + [`POST /v1/asr`](/features/speech-to-text) instead; it is simpler and returns + the whole transcript at once. + + +## Capture from a browser microphone + +The API key travels in a request header, and browsers cannot set headers on a WebSocket. Never put your API key in a web page: run a small relay on your server that adds the `Authorization` header and passes MessagePack frames through unchanged. The page then speaks the same protocol as the scripts above. + + + +```python relay.py +"""Browser <-> Fish Audio realtime speech-to-text relay. The API key stays on the server. + + pip install aiohttp + export FISH_API_KEY=... + python relay.py # then open http://localhost:8080 + +The page speaks the realtime protocol itself (MessagePack binary frames); the relay adds the +Authorization header and pipes binary frames both ways, unchanged. +""" +import asyncio +import os +from pathlib import Path + +from aiohttp import ClientSession, WSMsgType, WSServerHandshakeError, web + +URL = "wss://api.fish.audio/v1/asr/realtime" +API_KEY = os.environ["FISH_API_KEY"] + + +async def index(request): + return web.FileResponse(Path(__file__).with_name("index.html")) + + +async def relay(request): + browser = web.WebSocketResponse(max_msg_size=1 << 20) + await browser.prepare(request) + async with ClientSession() as http: + try: + # autoclose=False keeps the API's close code (1000, 4400, ...) readable below + upstream = await http.ws_connect( + URL, headers={"Authorization": f"Bearer {API_KEY}"}, autoclose=False + ) + except WSServerHandshakeError as e: # 401 / 402 / 403 / 429 / 503 ... + await browser.close(code=1013 if e.status in (429, 503) else 1011, message=f"HTTP {e.status}".encode()) + return browser + + async def pipe(src, dst): + async for msg in src: + if msg.type == WSMsgType.BINARY: + await dst.send_bytes(msg.data) + + to_api = asyncio.create_task(pipe(browser, upstream)) + to_browser = asyncio.create_task(pipe(upstream, browser)) + await asyncio.wait([to_api, to_browser], return_when=asyncio.FIRST_COMPLETED) + if to_api.done(): # the page went away (or sent audio_done and closed): let the session finish + await asyncio.wait([to_browser], timeout=10) + for task in (to_api, to_browser): + task.cancel() + code = upstream.close_code or 1000 + await upstream.close() + # forward the API's close code (1000, 1009, 1012, 44xx, 4500); 1006 and similar cannot be sent + await browser.close(code=code if code in (1000, 1009, 1011, 1012) or 4000 <= code < 5000 else 1011) + return browser + + +app = web.Application() +app.add_routes([web.get("/", index), web.get("/ws", relay)]) + +if __name__ == "__main__": + web.run_app(app, host="127.0.0.1", port=8080) +``` + +```html index.html + + +Realtime speech-to-text + +

+

+
+```
+
+
+
+Install `aiohttp`, put both files in one folder, run `python relay.py`, and open `http://localhost:8080`.
+
+- The page captures audio at the device's own sample rate and sends that rate in `config`; the server converts it, so the page does not resample. The rate must be one of the supported rates.
+- An `AudioWorklet` converts the samples to 16-bit PCM and posts about 100 ms per message.
+- **Stop** sends `audio_done`, so the server commits the last turn before it closes the connection.
+- The relay forwards the API's close code to the page. In production, add your own user authentication, per-user session limits and input checks to the relay.
+
+## Related
+
+- [Realtime Speech to Text](/features/speech-to-text/realtime)
+- [Build a realtime voice agent](/developer-guide/sdk-guide/cookbook/realtime-voice-agent)
+- [WebSocket Realtime Speech to Text reference](/api-reference/endpoint/websocket/asr-realtime)
diff --git a/developer-guide/sdk-guide/cookbook/realtime-voice-agent.mdx b/developer-guide/sdk-guide/cookbook/realtime-voice-agent.mdx
new file mode 100644
index 00000000..fe9ceca6
--- /dev/null
+++ b/developer-guide/sdk-guide/cookbook/realtime-voice-agent.mdx
@@ -0,0 +1,328 @@
+---
+title: "Build a realtime voice agent: streaming speech in, reply, speech out"
+description: "Use realtime speech to text to know when the user has finished, start replies early, keep talking through backchannels and stop on interruptions"
+icon: "headset"
+hidden: true
+noindex: true
+---
+
+import Prerequisites from "/snippets/prerequisites.mdx";
+
+
+  This guide is for building your own voice agent on the realtime speech-to-text
+  API. To have Fish Audio run the whole agent for you, see
+  [Agents](/agents/overview).
+
+
+
+  Realtime speech to text is in preview and is enabled per team. Contact
+  [support@fish.audio](mailto:support@fish.audio) to turn it on for your team.
+  Until then, connecting returns `403`.
+
+
+## Prerequisites
+
+
+
+Install a WebSocket client and a MessagePack library:
+
+```bash
+pip install "websockets>=14" msgpack
+```
+
+Read [Transcribe a live audio stream](/developer-guide/sdk-guide/cookbook/realtime-transcription) first if you have not used the realtime endpoint before.
+
+## How a conversation flows
+
+A voice agent has to answer three questions while audio streams in: has the user finished, is a sound the user talking to the agent at all, and is the user interrupting or just saying "mm-hm"? In the default `auto` mode, the server answers them with events. This is what the recipe below receives for a 15-second clip in which the user asks a question, says "uh-huh" while the agent answers, and then interrupts:
+
+| Elapsed | Server event                                                                                   | What the agent does                                                                             |
+| ------- | ---------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- |
+| 3.8 s   | `speech_start` #0                                                                              | Nothing yet: a turn opened on voice activity.                                                   |
+| 4.5 s   | `user_turn_taken` #0                                                                           | The user is speaking to the agent.                                                              |
+| 8.2 s   | `eager_end_of_turn` #0 "Can you tell me what time the pharmacy on Main Street closes tonight?" | Starts the LLM on this text.                                                                    |
+| 8.7 s   | `committed` #0, `user_turn: true`, same text                                                   | Reuses the reply it started, sends `agent_speaking: true` and the reply as context, and speaks. |
+| 10.9 s  | `speech_start` #1                                                                              | The agent is speaking: lowers its volume (ducks), in case this is an interruption.              |
+| 11.9 s  | `user_backchannel` #1                                                                          | "Uh-huh" is only an acknowledgement: restores the volume and keeps talking.                     |
+| 13.8 s  | `committed` #1, `user_turn: false`, `turn_kind: "backchannel"`                                 | Nothing to answer.                                                                              |
+| 14.2 s  | `speech_start` #2                                                                              | Ducks again.                                                                                    |
+| 14.7 s  | `user_turn_taken` #2                                                                           | The user is interrupting: stops speaking and sends `agent_speaking: false`.                     |
+
+Elapsed times are measured from the start of the script, which connects about 0.6 seconds before the audio starts.
+
+## Recipe
+
+The microphone is a WAV file streamed in real time, and `think()` and `speak()` are placeholders for your LLM and your text-to-speech playback. Everything else is the agent loop as you would ship it.
+
+```python voice_agent.py
+"""A voice-agent loop on Fish Audio realtime speech-to-text.
+
+    pip install "websockets>=14" msgpack
+    export FISH_API_KEY=...
+    python voice_agent.py speech.wav
+
+The "microphone" is a 16-bit mono WAV file streamed in real time, and think() / speak() are
+placeholders: replace them with your LLM and your text-to-speech playback.
+"""
+import asyncio
+import contextlib
+import os
+import re
+import sys
+import time
+import wave
+
+import msgpack
+import websockets
+
+URL = "wss://api.fish.audio/v1/asr/realtime"
+CHUNK_MS = 100
+TAGS = re.compile(r"<\|[^|]*\|>")  # speaker tags such as <|speaker:0|>
+T0 = time.monotonic()
+
+
+def log(*args):
+    print(f"{time.monotonic() - T0:6.2f}s", *args)
+
+
+def words(text):
+    """The transcript without speaker tags."""
+    return TAGS.sub("", text or "").strip()
+
+
+async def think(history):
+    """Placeholder LLM: replace with a call to your model."""
+    await asyncio.sleep(0.5)
+    return f"You asked: {history[-1]['content']} Let me check that for you, it will only take a moment."
+
+
+async def speak(text):
+    """Placeholder TTS: returns when the last sample has played."""
+    await asyncio.sleep(len(text) * 0.06)
+
+
+def duck(on):
+    """Placeholder: lower (or restore) the agent's playback volume."""
+    log("  duck" if on else "  unduck")
+
+
+class Agent:
+    def __init__(self, ws):
+        self.ws = ws
+        self.history = []     # the conversation, for your LLM
+        self.prepared = None  # (turn_index, task): a reply started on eager_end_of_turn
+        self.reply = None     # task: the reply being prepared or played
+        self.speaking = False
+        self.hung_up = False  # the caller's audio has ended: answer nothing more
+
+    async def send(self, message):
+        if message["type"] != "audio":
+            log("->", message)
+        await self.ws.send(msgpack.packb(message))
+
+    def start_thinking(self, text):
+        return asyncio.create_task(think(self.history + [{"role": "user", "content": text}]))
+
+    def drop_prepared(self):
+        if self.prepared:
+            self.prepared[1].cancel()
+            self.prepared = None
+
+    async def stop(self):
+        """Barge-in: stop the reply (its finally sends agent_speaking false)."""
+        if self.reply:
+            self.reply.cancel()
+            with contextlib.suppress(asyncio.CancelledError, websockets.ConnectionClosed):
+                await self.reply
+            self.reply = None
+
+    async def play(self, thinking):
+        """Speak a reply. Runs from committed until the last sample has played."""
+        try:
+            self.speaking = True
+            await self.send({"type": "agent_speaking", "speaking": True})  # committed to replying
+            text = await thinking  # the LLM may still be working: that is part of the reply
+            await self.send({"type": "context_add", "items": [{"role": "assistant", "text": text}]})
+            self.history.append({"role": "assistant", "content": text})
+            await speak(text)
+        finally:
+            self.speaking = False
+            await self.send({"type": "agent_speaking", "speaking": False})  # played out, or stopped
+
+    async def on_event(self, m):
+        t = m["type"]
+        if t == "eager_end_of_turn":
+            self.drop_prepared()
+            if words(m["text"]):  # start the reply early; committed will have the same text
+                self.prepared = (m["turn_index"], self.start_thinking(words(m["text"])))
+        elif t == "turn_resumed":
+            self.drop_prepared()  # the user went on: that text is not final
+        elif t == "speech_start":
+            if self.speaking:
+                duck(True)  # maybe an interruption, maybe "mm-hm"
+        elif t == "user_backchannel":
+            duck(False)  # the user only acknowledged: keep talking
+        elif t == "user_turn_taken":
+            await self.stop()  # the user is taking the turn: stop talking
+        elif t == "committed":
+            text = words(m["text"])
+            if m.get("user_turn", True) and text and not self.hung_up:
+                if self.prepared and self.prepared[0] == m["turn_index"]:
+                    thinking = self.prepared[1]  # reuse the reply started on eager_end_of_turn
+                    self.prepared = None
+                else:
+                    self.drop_prepared()
+                    thinking = self.start_thinking(text)
+                self.history.append({"role": "user", "content": text})
+                await self.stop()
+                self.reply = asyncio.create_task(self.play(thinking))
+            else:  # not the user's turn (noise, another voice, a backchannel), or no words
+                if self.prepared and self.prepared[0] == m["turn_index"]:
+                    self.drop_prepared()
+                if self.speaking:
+                    duck(False)
+
+
+async def microphone(agent, wav, rate):
+    """Stream the file in real time, then end the session."""
+    loop = asyncio.get_running_loop()
+    start, sent_ms = loop.time(), 0
+    while chunk := wav.readframes(rate * CHUNK_MS // 1000):
+        await agent.send({"type": "audio", "data": chunk})
+        sent_ms += CHUNK_MS
+        await asyncio.sleep(max(0.0, start + sent_ms / 1000 - loop.time()))
+    agent.hung_up = True
+    await agent.stop()  # agent_speaking false goes before audio_done
+    await agent.send({"type": "audio_done"})
+
+
+async def main(path):
+    wav = wave.open(path, "rb")
+    rate = wav.getframerate()
+    headers = {"Authorization": f"Bearer {os.environ['FISH_API_KEY']}"}
+    async with websockets.connect(URL, additional_headers=headers) as ws:
+        agent = Agent(ws)
+        await agent.send({
+            "type": "config",
+            "audio": {"format": "pcm_s16le", "sample_rate": rate, "channels": 1},
+            "turn_detection": {"mode": "auto", "eager": True},
+            "context": {"terms": ["Main Street"]},
+        })
+        mic = None
+        try:
+            async for frame in ws:
+                m = msgpack.unpackb(frame)
+                shown = {k: v for k, v in m.items() if k in ("turn_index", "text", "user_turn", "turn_kind", "reason")}
+                log("<-", m["type"], shown)
+                if m["type"] == "session_started":
+                    mic = asyncio.create_task(microphone(agent, wav, rate))
+                await agent.on_event(m)
+        finally:
+            if mic:
+                mic.cancel()
+            await agent.stop()
+    log("closed", ws.close_code)
+
+
+if __name__ == "__main__":
+    asyncio.run(main(sys.argv[1]))
+```
+
+Run it with your API key in `FISH_API_KEY`:
+
+```bash
+python voice_agent.py speech.wav
+```
+
+The recipe keeps a reply's full text in its history even when the user interrupts it. In production, keep only the part the user heard.
+
+## Decide when to answer
+
+Answer a turn when `committed.user_turn` is `true` and its text, without speaker tags, is not empty. Do not decide by `reason`: a turn can end for several reasons, and non-user turns end with `eot` too. If `user_turn` is ever missing, treat it as `true`.
+
+A turn with `user_turn: false` is not the user talking to your agent: usually background noise, a TV or another person nearby (`turn_kind: "other"`), or an acknowledgement while your agent was speaking (`turn_kind: "backchannel"`). Its text can be shown dimmed in a transcript, or left out.
+
+In `auto` mode, short pauses in the middle of a sentence usually do not end the turn: the server judges from what was said and how whether the speaker has finished. A speaker who stops mid-sentence and does not go on has the turn ended about 2 seconds later. `silence_timeout_ms` (default 5000) is only a safety net.
+
+## Start early with eager end of turn
+
+With `turn_detection.eager: true`, the server sends `eager_end_of_turn` once the turn has probably ended, before `committed`. Its text is exactly what `committed` will say unless `turn_resumed` follows, so you can start your LLM on it:
+
+- On `eager_end_of_turn` with non-empty text, start preparing a reply. Do not send `agent_speaking: true` until you start playing it; waiting for `committed` before you play is the safe default.
+- On `turn_resumed`, the user went on: cancel the prepared reply. A new `eager_end_of_turn` can follow.
+- On `committed`, if you prepared a reply for this turn, use it; otherwise start one now.
+
+Not every turn has an `eager_end_of_turn`, and it can come right before `committed`, so your agent must also work without it. In the clip above, the eager signal came about half a second before the final transcript.
+
+## Tell the server when your agent speaks
+
+Send `agent_speaking` so the server can tell an acknowledgement from an interruption:
+
+- Send `{"type": "agent_speaking", "speaking": true}` when you commit to playing a reply: on `committed`, or, if you play a reply before `committed`, when playback starts. The wait for your speech synthesis and the gaps between sentences count as speaking. If the first audio is more than about 2 seconds away (a slow LLM or a tool call), treat the wait as a pause, as below.
+- Send `{"type": "agent_speaking", "speaking": false}` when the last audio sample has played to the user, not when synthesis finishes, or when you stop the reply. If `turn_resumed` cancels a reply that was already playing, stop it and send `false`.
+- If a reply pauses for more than about 2 seconds, for example while a tool call runs, send `false` for the pause and `true` when it goes on.
+- Send it on the same connection as your audio, at the moment the state changes, and only when it changes. It applies at its position in the audio stream.
+
+Without `agent_speaking`, every acknowledgement takes the turn, and you have to tell "mm-hm" from an interruption yourself.
+
+## Handle interruptions and backchannels
+
+While your agent is speaking:
+
+1. On `speech_start`, duck the agent's audio: the user may be interrupting. For the earliest signal, turn on `results.vad_events` and duck on `vad` messages instead.
+2. On `user_backchannel`, the user only acknowledged ("mm-hm", "yeah", "right", "uh-huh"): restore the volume and keep talking.
+3. On `user_turn_taken`, the user is taking the turn: stop the agent and send `agent_speaking: false`.
+4. On `committed` with `user_turn: false`, restore the volume.
+5. Cap any duck on your side to a few seconds: a turn over continuous background noise can last up to 20 seconds. When the user starts talking over such noise, `user_turn_taken` can come a few hundred milliseconds later than usual.
+
+In `vad` and `manual` mode there is no `user_turn_taken` or `user_backchannel`: stop on `speech_start` instead.
+
+## Give the model context
+
+The server uses the conversation to recognize names and follow-ups:
+
+- **Custom vocabulary:** send names, products and jargon in `context.terms` (up to 100 terms of up to 32 characters). Replace the list during the session with `configure`, for example after you look up the caller's account.
+- **What your agent said:** send each line your agent speaks as an `assistant` item with `context_add`, right after the `committed` it answers. The recipe does this when a reply starts.
+- **Conversation history:** committed user turns are added automatically; non-user turns are not. Speaker labels stay the same for the whole session.
+- **Prior conversation:** to continue a conversation in a new session, pass earlier lines in `context.history` in `config`.
+
+```json
+{
+  "type": "config",
+  "audio": { "format": "pcm_s16le", "sample_rate": 16000, "channels": 1 },
+  "turn_detection": { "mode": "auto", "eager": true },
+  "context": {
+    "history": [
+      {
+        "role": "assistant",
+        "text": "Thanks for calling Main Street Pharmacy."
+      },
+      { "role": "user", "text": "Hi, I'm calling about my prescription." }
+    ],
+    "terms": ["Main Street Pharmacy", "Lisinopril"]
+  }
+}
+```
+
+## Capture audio well
+
+- **Keep echo cancellation on.** Without it, the agent's own voice reaches the microphone, opens turns and can be taken for the user. In a browser, request `echoCancellation: true` from `getUserMedia`.
+- **Turn automatic gain control off where you can** (`autoGainControl: false` in a browser), so quiet background sound is not amplified between phrases.
+- **Test with headphones first.** They keep the agent's voice out of the microphone, so you can check turn-taking before you tune echo cancellation.
+- **Send audio as you capture it**, mono 16-bit PCM in 80–100 ms messages, at the device's own rate if it is one of the supported rates. The server converts the rate, so do not resample. Telephony audio at 8 kHz is supported; decode μ-law or A-law call audio to 16-bit PCM first.
+- **Keep keys on your server.** In a browser, capture audio and relay it through your server, as in [Capture from a browser microphone](/developer-guide/sdk-guide/cookbook/realtime-transcription#capture-from-a-browser-microphone). Send `agent_speaking` on that same connection so it stays in order with the audio.
+
+## Languages
+
+Turn detection, including eager end of turn, backchannel detection and background awareness, is trained on a set of major languages. Transcription and language identification cover many more languages. In languages outside that set, a turn ends about 2 seconds after the speaker stops and there is no `eager_end_of_turn`. For faster turns in those languages, use `vad` mode with a shorter `silence_timeout_ms`, or end turns yourself with `finalize`.
+
+## Connect your LLM and speech
+
+Replace `think()` with a call to your LLM, and `speak()` with text to speech and playback. To start speaking before the LLM has finished, stream its tokens into the [TTS WebSocket](/developer-guide/sdk-guide/cookbook/realtime-llm-to-speech) and play audio chunks as they arrive. Send `agent_speaking: false` when the last chunk has played, not when synthesis ends.
+
+## Related
+
+- [Realtime Speech to Text](/features/speech-to-text/realtime)
+- [Transcribe a live audio stream](/developer-guide/sdk-guide/cookbook/realtime-transcription)
+- [WebSocket Realtime Speech to Text reference](/api-reference/endpoint/websocket/asr-realtime)
+- [Realtime: LLM tokens → speech](/developer-guide/sdk-guide/cookbook/realtime-llm-to-speech)
diff --git a/features/speech-to-text/realtime.mdx b/features/speech-to-text/realtime.mdx
new file mode 100644
index 00000000..3928fd3a
--- /dev/null
+++ b/features/speech-to-text/realtime.mdx
@@ -0,0 +1,236 @@
+---
+title: "Realtime Speech to Text"
+sidebarTitle: "Realtime"
+description: "Transcribe live audio turn by turn with transcribe-1-pro-realtime: turn detection that knows when the user has finished, backchannel and background awareness, and conversation context"
+icon: "microphone-lines"
+hidden: true
+noindex: true
+---
+
+Stream live audio over a WebSocket and get the transcript turn by turn while people speak. `transcribe-1-pro-realtime` does more than transcribe: it decides when the user has finished their turn, usually tells the user talking to your app apart from background sound, and notices when a "mm-hm" is only an acknowledgement. This helps your voice agent decide when to answer, when to keep talking and when to stop. Connect to `wss://api.fish.audio/v1/asr/realtime`, send audio as you capture it, and read the events the server sends back.
+
+
+  Realtime speech to text is in preview and is enabled per team. Contact
+  [support@fish.audio](mailto:support@fish.audio) to turn it on for your team.
+  Until then, connecting returns `403`.
+
+
+
+  
+    Every message, field, error and limit.
+  
+  
+    Stream a file or a browser microphone.
+  
+  
+    Turn-taking, interruptions and backchannels.
+  
+
+
+## When to use it
+
+
+  
+    Know when the user has finished, start replies early, and handle
+    interruptions.
+  
+  
+    Show live drafts as people speak, then the final text for each turn.
+  
+  
+    Act on what the user said as soon as the turn ends, or on button release
+    with push-to-talk.
+  
+  
+    Stream 8 kHz call audio as it arrives. Decode μ-law or A-law audio to 16-bit
+    PCM first; the server converts the sample rate.
+  
+
+
+To transcribe a recording you already have, use [`POST /v1/asr`](/features/speech-to-text) with `transcribe-1-pro` instead:
+
+|                | Realtime (`/v1/asr/realtime`)                                  | File (`POST /v1/asr`)                                      |
+| -------------- | -------------------------------------------------------------- | ---------------------------------------------------------- |
+| Model          | `transcribe-1-pro-realtime`                                    | `transcribe-1-pro`                                         |
+| Input          | Live 16-bit PCM over a WebSocket                               | A complete audio file (WAV, MP3, Opus, and more)           |
+| Output         | Events per turn: drafts, turn signals and the final transcript | One transcript, with optional timestamps and speaker turns |
+| Turn detection | Yes                                                            | No                                                         |
+| Session length | Up to 60 minutes                                               | Recordings up to 60 minutes                                |
+
+The realtime endpoints of the [compatible APIs](/developer-guide/compat/capabilities) are separate from this endpoint. They use manual commit and have no server-side turn detection.
+
+## Quick start
+
+The SDKs do not wrap this endpoint; you connect with any WebSocket client and exchange MessagePack messages.
+
+
+  
+    Open `wss://api.fish.audio/v1/asr/realtime` from your server with the header
+    `Authorization: Bearer `.
+  
+  
+    Send a `config` message with your audio format and options, and wait for
+    `session_started`.
+  
+  
+    Send `audio` messages with 16-bit mono PCM, 80–100 ms each, as you capture
+    it.
+  
+  
+    Each turn ends with one `committed` message holding its final text. Answer
+    it if `user_turn` is `true`.
+  
+  
+    Send `audio_done`. The server commits the last turn, sends `session_done`
+    and closes the connection.
+  
+
+
+```json
+{
+  "type": "config",
+  "audio": { "format": "pcm_s16le", "sample_rate": 16000, "channels": 1 },
+  "turn_detection": { "mode": "auto", "eager": true },
+  "results": { "drafts": true }
+}
+```
+
+[Transcribe a live audio stream](/developer-guide/sdk-guide/cookbook/realtime-transcription) has complete, tested clients in Python and Node.js, and a browser microphone page with a server relay.
+
+## How a session works
+
+Audio is divided into turns. A turn opens when voice activity begins (`speech_start`) and ends with exactly one `committed` message, which holds the final transcript and says why the turn ended. In between, you can receive live `draft` text and turn signals:
+
+```text
+speech_start → draft … → user_turn_taken → draft … → eager_end_of_turn → committed
+```
+
+- `user_turn_taken`: the server judged that the user is taking their turn.
+- `eager_end_of_turn`: the turn has probably ended; its text equals the final transcript unless `turn_resumed` follows.
+- `committed`: the final transcript. `user_turn` says whether it was the user's turn.
+
+## Smart Turn Detection
+
+In the default `auto` mode, the server's turn detection decides when a turn ends, from what was said and how, not from silence alone.
+
+### Eager end of turn
+
+With `turn_detection.eager: true`, you receive `eager_end_of_turn` as soon as the turn has probably ended, before the final transcript. Its text is exactly the text `committed` will have, unless `turn_resumed` follows because the user went on. A voice agent can start its LLM on the eager text and save that time.
+
+### Backchannel detection
+
+Tell the server when your agent is speaking with `agent_speaking`. While it speaks, a short "mm-hm", "yeah", "right" or "uh-huh" from the user produces `user_backchannel`: the agent keeps talking. A real interruption produces `user_turn_taken`: the agent stops.
+
+### Background awareness
+
+Noise, a TV or side conversations usually become non-user turns, committed with `user_turn: false`, so your agent does not answer them.
+
+### Room to think
+
+Short pauses in the middle of a sentence usually do not end the turn, so people can stop to think. When a speaker stops mid-sentence and does not go on, the turn ends about 2 seconds later. As a further safety net, a user turn with no voice activity for `silence_timeout_ms` (default 5 seconds) always ends.
+
+## History Aware Transcription
+
+The server uses the conversation to recognize names and follow-ups.
+
+- **Custom vocabulary:** list names, products and jargon in `context.terms`, up to 100 terms of up to 32 characters. Update the list during the session with `configure`.
+- **What the agent just said:** send each line your agent speaks with `context_add` as an `assistant` item, so the user's answer is heard in context.
+- **Conversation history:** recent turns of the session are used as context, and speaker labels such as `<|speaker:0|>` stay the same for the whole session.
+- **Prior conversation:** start a session with earlier lines in `context.history`.
+
+## Turn detection modes
+
+| Mode             | A turn ends when                                                   | Use it for                             |
+| ---------------- | ------------------------------------------------------------------ | -------------------------------------- |
+| `auto` (default) | Turn detection judges that the user has finished.                  | Voice agents and conversations.        |
+| `vad`            | A fixed silence has passed (`silence_timeout_ms`, default 700 ms). | Simple silence-based endpointing.      |
+| `manual`         | You send `finalize`.                                               | Push-to-talk, or your own endpointing. |
+
+Eager end of turn, backchannel detection and background awareness are available in `auto` mode, when the session starts in `auto` (the default). In `vad` and `manual` mode, and after a later switch to `auto` from another mode, every turn is a user turn. See [Turn detection modes](/api-reference/endpoint/websocket/asr-realtime#turn-detection-modes) for details.
+
+## Languages
+
+Smart Turn Detection is trained on a set of major languages. Transcription covers many more languages, and `draft`, `eager_end_of_turn` and `committed` carry the language of their own text as `language_code` when it can be identified, one of 29 [supported codes](/api-reference/endpoint/websocket/asr-realtime#language).
+
+In languages outside that set, a turn ends about 2 seconds after the speaker stops, and there is no eager signal. For faster turns in those languages, use `vad` mode, or end turns yourself with `finalize`.
+
+## Latency
+
+Measured on the production service with synthetic speech:
+
+- The first text, a `draft` (with `results.drafts: true`), arrives about 0.5 seconds after speech starts.
+- `eager_end_of_turn` arrives about 0.6 seconds after the speaker stops.
+- `committed` arrives about 1.1 seconds after the speaker stops.
+
+Your network and how you send audio add to these times. Send audio as you capture it, in 80–100 ms messages.
+
+## Implementation details
+
+### Audio
+
+Send 16-bit little-endian PCM, mono, at 8000, 12000, 16000, 22050, 24000, 32000, 44100 or 48000 Hz. The server converts the sample rate, so you never need to resample. Compressed formats such as Opus or MP3 are not accepted on this endpoint.
+
+### Speakers
+
+Transcripts contain speaker tags such as `<|speaker:0|>`, stable within a session. Strip them, or send `results.diarize: false` for plain text.
+
+### Keys and browsers
+
+Your API key travels in a request header. Connect from your server, never from a web page. To transcribe a browser microphone, relay the audio through your server, as shown in the [quickstart](/developer-guide/sdk-guide/cookbook/realtime-transcription#capture-from-a-browser-microphone).
+
+### Limits
+
+A session lasts up to 60 minutes and a turn up to 60 seconds. Each open connection holds one of your team's concurrent session slots. See [Limits](/api-reference/endpoint/websocket/asr-realtime#limits) for the full list.
+
+## Going further
+
+
+  
+    The agent loop: eager replies, interruptions and backchannels.
+  
+  
+    Speak your agent's replies with streaming text to speech.
+  
+  
+    Recordings up to 60 minutes with `transcribe-1-pro`.
+  
+  
+    Messages, errors, close codes and limits.
+  
+
diff --git a/tests/cookbooks/requirements.txt b/tests/cookbooks/requirements.txt
index 3b17e31d..8c9c7f96 100644
--- a/tests/cookbooks/requirements.txt
+++ b/tests/cookbooks/requirements.txt
@@ -2,3 +2,5 @@ fish-audio-sdk>=1.3.0
 httpx>=0.28
 python-dotenv>=1.0
 pytest>=8.0
+websockets>=14
+msgpack>=1.0
diff --git a/tests/cookbooks/specs.py b/tests/cookbooks/specs.py
index d58b7be7..1c15d741 100644
--- a/tests/cookbooks/specs.py
+++ b/tests/cookbooks/specs.py
@@ -8,6 +8,26 @@
 
 COOKBOOK = "developer-guide/sdk-guide/cookbook"
 
+# Runs a realtime recipe's main() on speech.wav, capturing what it prints in `_lines`.
+REALTIME_RUN = """
+import asyncio, builtins
+import pytest, websockets
+_lines = []
+def print(*args, **kwargs):
+    _lines.append(" ".join(str(a) for a in args))
+    builtins.print(*args, **kwargs)
+try:
+    asyncio.run(main("speech.wav"))
+except SystemExit as e:
+    if "HTTP 403" in str(e):
+        pytest.skip("realtime speech to text is not enabled for this key's team")
+    raise
+except websockets.InvalidStatus as e:
+    if e.response.status_code == 403:
+        pytest.skip("realtime speech to text is not enabled for this key's team")
+    raise
+"""
+
 SPECS = [
     {
         "slug": "streaming-to-file",
@@ -123,4 +143,29 @@
              "subs": {'""': f'"{PUBLIC_VOICE}"'}},
         ],
     },
+    # Realtime speech to text (WebSocket). The recipes are scripts: the postamble runs
+    # main() on the fixture clip and checks the printed events. The endpoint is enabled
+    # per team, so a key without access (HTTP 403) skips instead of failing.
+    {
+        "slug": "realtime-transcription",
+        "path": f"{COOKBOOK}/realtime-transcription.mdx",
+        "cases": [
+            {"name": "stream a wav file", "block": 0, "postamble": REALTIME_RUN + """
+assert any(line.startswith("session ") for line in _lines), _lines
+assert any(line.startswith("committed") for line in _lines), _lines
+assert any(line.startswith("done: ") for line in _lines), _lines
+assert any(line.startswith("closed: 1000") for line in _lines), _lines
+"""},
+        ],
+    },
+    {
+        "slug": "realtime-voice-agent",
+        "path": f"{COOKBOOK}/realtime-voice-agent.mdx",
+        "cases": [
+            {"name": "agent loop on a wav file", "block": 0, "postamble": REALTIME_RUN + """
+assert any("<- committed" in line for line in _lines), _lines
+assert any("closed 1000" in line for line in _lines), _lines
+"""},
+        ],
+    },
 ]
diff --git a/tests/js/package.json b/tests/js/package.json
index 55b6ea26..5e61e09f 100644
--- a/tests/js/package.json
+++ b/tests/js/package.json
@@ -8,6 +8,8 @@
     "test": "node run.mjs"
   },
   "dependencies": {
-    "fish-audio": "^0.1.0"
+    "fish-audio": "^0.1.0",
+    "@msgpack/msgpack": "^3.0.0",
+    "ws": "^8.18.0"
   }
 }
diff --git a/tests/js/specs.mjs b/tests/js/specs.mjs
index 9037e7f6..cab219c1 100644
--- a/tests/js/specs.mjs
+++ b/tests/js/specs.mjs
@@ -14,4 +14,5 @@ export const SPECS = [
   { slug: "oneshot-vs-persistent-cloning", mdx: "developer-guide/sdk-guide/cookbook/oneshot-vs-persistent-cloning.mdx", cases: [{ name: "primary", block: 0, file: ["oneshot.mp3", "mp3"] }] },
   { slug: "discover-library-voice", mdx: "developer-guide/sdk-guide/cookbook/discover-library-voice.mdx", cases: [{ name: "primary", block: 0, file: ["out.mp3", "mp3"], subs: { "": "" } }] },
   { slug: "voice-agent-loop", mdx: "developer-guide/sdk-guide/cookbook/voice-agent-loop.mdx", cases: [{ name: "primary", block: 0, file: ["reply.mp3", "mp3"], subs: { "": "" } }] },
+  { slug: "realtime-transcription", mdx: "developer-guide/sdk-guide/cookbook/realtime-transcription.mdx", cases: [{ name: "stream a wav file", block: 0, subs: { "process.argv[2]": '"speech.wav"' } }] },
 ];